{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "这里用Pytorch搭建AFM Model在cretio数据集上进行点击率预测的任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:34.021727Z",
     "start_time": "2021-01-16T14:15:25.036877Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"导入包\"\"\"\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import datetime\n",
    "\n",
    "# pytorch\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:38.488532Z",
     "start_time": "2021-01-16T14:15:38.477560Z"
    }
   },
   "outputs": [],
   "source": [
    "file_path = './preprocessed_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:41.280858Z",
     "start_time": "2021-01-16T14:15:41.271918Z"
    }
   },
   "outputs": [],
   "source": [
    "def prepared_data(file_path):\n",
    "    # 读入训练集， 验证集和测试集\n",
    "    train = pd.read_csv(file_path + 'train_set.csv')\n",
    "    val = pd.read_csv(file_path + 'val_set.csv')\n",
    "    test = pd.read_csv(file_path + 'test_set.csv')\n",
    "    \n",
    "    trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values\n",
    "    val_x, val_y = val.drop(columns='Label').values, val['Label'].values\n",
    "    test_x = test.values\n",
    "    \n",
    "    fea_col = np.load(file_path + 'fea_col.npy', allow_pickle=True)\n",
    "    \n",
    "    return fea_col, (trn_x, trn_y), (val_x, val_y), test_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:45.176474Z",
     "start_time": "2021-01-16T14:15:45.136207Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"导入数据\"\"\"\n",
    "fea_cols, (trn_x, trn_y), (val_x, val_y), test_x = prepared_data(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:53.691859Z",
     "start_time": "2021-01-16T14:15:53.671911Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(trn_x).float(), torch.tensor(trn_y).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_x).float(), torch.tensor(val_y).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=32)\n",
    "dl_val = DataLoader(dl_val_dataset, shuffle=True, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-16T14:15:55.523665Z",
     "start_time": "2021-01-16T14:15:55.491755Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 39]) tensor([1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1.])\n"
     ]
    }
   ],
   "source": [
    "# 看一眼数据\n",
    "for x, y in iter(dl_train):\n",
    "    print(x.shape, y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建模型\n",
    "这里依然是使用继承nn.Modult基类构建模型， 并辅助应用模型容器进行封装， AFM的模型架构如下：\n",
    "\n",
    "![](https://img-blog.csdnimg.cn/20210102204934171.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d1emhvbmdxaWFuZw==,size_1,color_FFFFFF,t_70#pic_center)\n",
    "\n",
    "这个模型在NFM的基础上加入了Attention机制， DNN模块没有改变， 我们需要实现这个Attention网络， 然后拼起来构成AFMmoxi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:11:02.239154Z",
     "start_time": "2020-12-28T14:11:02.232172Z"
    }
   },
   "outputs": [],
   "source": [
    "class Dnn(nn.Module):\n",
    "    def __init__(self, hidden_units, dropout=0.):\n",
    "        \"\"\"\n",
    "        hidden_units: 列表， 每个元素表示每一层的神经单元个数， 比如[256, 128, 64], 两层网络， 第一层神经单元128， 第二层64， 第一个维度是输入维度\n",
    "        dropout = 0.\n",
    "        \"\"\"\n",
    "        super(Dnn, self).__init__()\n",
    "        \n",
    "        self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1], hidden_units[1:]))])\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, x):  \n",
    "        for linear in self.dnn_network:\n",
    "            x = linear(x)\n",
    "            x = F.relu(x)    \n",
    "        x = self.dropout(x) \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:10.543151Z",
     "start_time": "2020-12-28T14:40:10.519174Z"
    }
   },
   "outputs": [],
   "source": [
    "class NFM(nn.Module):\n",
    "    def __init__(self, feature_columns, hidden_units, dnn_dropout=0.):\n",
    "        \"\"\"\n",
    "        NFM:\n",
    "        :param feature_columns: 特征信息， 这个传入的是fea_cols\n",
    "        :param hidden_units: 隐藏单元个数， 一个列表的形式， 列表的长度代表层数， 每个元素代表每一层神经元个数\n",
    "        \"\"\"\n",
    "        super(NFM, self).__init__()\n",
    "        self.dense_feature_cols, self.sparse_feature_cols = feature_columns\n",
    "        \n",
    "        # embedding\n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(i): nn.Embedding(num_embeddings=feat['feat_num'], embedding_dim=feat['embed_dim'])\n",
    "            for i, feat in enumerate(self.sparse_feature_cols)\n",
    "        })\n",
    "        \n",
    "        # 这里要注意Pytorch的linear和tf的dense的不同之处， 前者的linear需要输入特征和输出特征维度， 而传入的hidden_units的第一个是第一层隐藏的神经单元个数，这里需要加个输入维度\n",
    "        self.fea_num = len(self.dense_feature_cols) + self.sparse_feature_cols[0]['embed_dim']\n",
    "        hidden_units.insert(0, self.fea_num)\n",
    "        \n",
    "        self.bn = nn.BatchNorm1d(self.fea_num)     \n",
    "        self.dnn_network = Dnn(hidden_units, dnn_dropout)\n",
    "        self.nn_final_linear = nn.Linear(hidden_units[-1], 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_inputs, sparse_inputs = x[:, :len(self.dense_feature_cols)], x[:, len(self.dense_feature_cols):]\n",
    "        sparse_inputs = sparse_inputs.long()       # 转成long类型才能作为nn.embedding的输入\n",
    "        sparse_embeds = [self.embed_layers['embed_'+str(i)](sparse_inputs[:, i]) for i in range(sparse_inputs.shape[1])]\n",
    "        sparse_embeds = torch.stack(sparse_embeds)     # embedding堆起来， (field_dim, None, embed_dim)\n",
    "        sparse_embeds = sparse_embeds.permute((1, 0, 2))\n",
    "        # 这里得到embedding向量之后 sparse_embeds(None, field_num, embed_dim), 进行特征交叉层，按照那个公式\n",
    "        embed_cross = 1/2 * (\n",
    "            torch.pow(torch.sum(sparse_embeds, dim=1),2) - torch.sum(torch.pow(sparse_embeds, 2), dim=1)\n",
    "        )  # (None, embed_dim)\n",
    "        \n",
    "        # 把离散特征和连续特征进行拼接作为FM和DNN的输入\n",
    "        x = torch.cat([embed_cross, dense_inputs], dim=-1)\n",
    "        # BatchNormalization\n",
    "        x = self.bn(x)\n",
    "        # deep\n",
    "        dnn_outputs = self.nn_final_linear(self.dnn_network(x))\n",
    "        outputs = F.sigmoid(dnn_outputs)\n",
    "        \n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:12.065615Z",
     "start_time": "2020-12-28T14:40:12.045625Z"
    }
   },
   "outputs": [],
   "source": [
    "# 建立模型\n",
    "hidden_units = [128, 64, 32]\n",
    "dnn_dropout = 0.\n",
    "\n",
    "model = NFM(fea_cols, hidden_units, dnn_dropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:12.938762Z",
     "start_time": "2020-12-28T14:40:12.929786Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                    [-1, 8]             632\n",
      "         Embedding-2                    [-1, 8]           2,016\n",
      "         Embedding-3                    [-1, 8]          10,344\n",
      "         Embedding-4                    [-1, 8]           8,344\n",
      "         Embedding-5                    [-1, 8]             240\n",
      "         Embedding-6                    [-1, 8]              56\n",
      "         Embedding-7                    [-1, 8]           9,312\n",
      "         Embedding-8                    [-1, 8]             312\n",
      "         Embedding-9                    [-1, 8]              16\n",
      "        Embedding-10                    [-1, 8]           7,264\n",
      "        Embedding-11                    [-1, 8]           7,408\n",
      "        Embedding-12                    [-1, 8]           9,912\n",
      "        Embedding-13                    [-1, 8]           6,592\n",
      "        Embedding-14                    [-1, 8]             160\n",
      "        Embedding-15                    [-1, 8]           6,552\n",
      "        Embedding-16                    [-1, 8]           9,272\n",
      "        Embedding-17                    [-1, 8]              72\n",
      "        Embedding-18                    [-1, 8]           4,272\n",
      "        Embedding-19                    [-1, 8]           1,608\n",
      "        Embedding-20                    [-1, 8]              32\n",
      "        Embedding-21                    [-1, 8]           9,632\n",
      "        Embedding-22                    [-1, 8]              56\n",
      "        Embedding-23                    [-1, 8]              96\n",
      "        Embedding-24                    [-1, 8]           5,832\n",
      "        Embedding-25                    [-1, 8]             264\n",
      "        Embedding-26                    [-1, 8]           4,432\n",
      "      BatchNorm1d-27                   [-1, 21]              42\n",
      "           Linear-28                  [-1, 128]           2,816\n",
      "           Linear-29                   [-1, 64]           8,256\n",
      "           Linear-30                   [-1, 32]           2,080\n",
      "          Dropout-31                   [-1, 32]               0\n",
      "           Linear-32                    [-1, 1]              33\n",
      "================================================================\n",
      "Total params: 117,955\n",
      "Trainable params: 117,955\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000149\n",
      "Forward/backward pass size (MB): 0.003708\n",
      "Params size (MB): 0.449963\n",
      "Estimated Total Size (MB): 0.453819\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, input_shape=(trn_x.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:14.812806Z",
     "start_time": "2020-12-28T14:40:14.802833Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.5276],\n",
      "        [0.5210],\n",
      "        [0.5334],\n",
      "        [0.5276],\n",
      "        [0.5274],\n",
      "        [0.5255],\n",
      "        [0.5185],\n",
      "        [0.5249],\n",
      "        [0.5266],\n",
      "        [0.5279],\n",
      "        [0.5309],\n",
      "        [0.5319],\n",
      "        [0.5191],\n",
      "        [0.5226],\n",
      "        [0.5222],\n",
      "        [0.5302],\n",
      "        [0.5301],\n",
      "        [0.5364],\n",
      "        [0.5357],\n",
      "        [0.5211],\n",
      "        [0.5220],\n",
      "        [0.5234],\n",
      "        [0.5291],\n",
      "        [0.5290],\n",
      "        [0.5303],\n",
      "        [0.5292],\n",
      "        [0.5203],\n",
      "        [0.5268],\n",
      "        [0.5253],\n",
      "        [0.5223],\n",
      "        [0.5198],\n",
      "        [0.5310]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    out = model(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:17.515532Z",
     "start_time": "2020-12-28T14:40:17.500621Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关配置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = optim.Adam(params=model.parameters(), lr=0.001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:28.839832Z",
     "start_time": "2020-12-28T14:40:21.243131Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_training.........\n",
      "================================================================2020-12-28 22:40:21\n",
      "[step=10] loss: 0.680, auc: 0.527\n",
      "[step=20] loss: 0.631, auc: 0.500\n",
      "[step=30] loss: 0.611, auc: 0.495\n",
      "[step=40] loss: 0.598, auc: 0.508\n",
      "\n",
      "EPOCH=1, loss=0.598, auc = 0.508, val_loss=0.482, val_auc = 0.616\n",
      "\n",
      "================================================================================2020-12-28 22:40:22\n",
      "[step=10] loss: 0.534, auc: 0.611\n",
      "[step=20] loss: 0.536, auc: 0.632\n",
      "[step=30] loss: 0.522, auc: 0.653\n",
      "[step=40] loss: 0.505, auc: 0.661\n",
      "\n",
      "EPOCH=2, loss=0.505, auc = 0.661, val_loss=0.435, val_auc = 0.658\n",
      "\n",
      "================================================================================2020-12-28 22:40:22\n",
      "[step=10] loss: 0.468, auc: 0.734\n",
      "[step=20] loss: 0.472, auc: 0.718\n",
      "[step=30] loss: 0.482, auc: 0.710\n",
      "[step=40] loss: 0.488, auc: 0.697\n",
      "\n",
      "EPOCH=3, loss=0.488, auc = 0.697, val_loss=0.446, val_auc = 0.633\n",
      "\n",
      "================================================================================2020-12-28 22:40:23\n",
      "[step=10] loss: 0.494, auc: 0.724\n",
      "[step=20] loss: 0.497, auc: 0.716\n",
      "[step=30] loss: 0.484, auc: 0.709\n",
      "[step=40] loss: 0.481, auc: 0.713\n",
      "\n",
      "EPOCH=4, loss=0.481, auc = 0.713, val_loss=0.436, val_auc = 0.602\n",
      "\n",
      "================================================================================2020-12-28 22:40:24\n",
      "[step=10] loss: 0.492, auc: 0.684\n",
      "[step=20] loss: 0.462, auc: 0.734\n",
      "[step=30] loss: 0.469, auc: 0.734\n",
      "[step=40] loss: 0.474, auc: 0.725\n",
      "\n",
      "EPOCH=5, loss=0.474, auc = 0.725, val_loss=0.443, val_auc = 0.662\n",
      "\n",
      "================================================================================2020-12-28 22:40:25\n",
      "[step=10] loss: 0.423, auc: 0.740\n",
      "[step=20] loss: 0.434, auc: 0.756\n",
      "[step=30] loss: 0.438, auc: 0.763\n",
      "[step=40] loss: 0.462, auc: 0.750\n",
      "\n",
      "EPOCH=6, loss=0.462, auc = 0.750, val_loss=0.463, val_auc = 0.630\n",
      "\n",
      "================================================================================2020-12-28 22:40:25\n",
      "[step=10] loss: 0.444, auc: 0.757\n",
      "[step=20] loss: 0.458, auc: 0.755\n",
      "[step=30] loss: 0.466, auc: 0.748\n",
      "[step=40] loss: 0.457, auc: 0.758\n",
      "\n",
      "EPOCH=7, loss=0.457, auc = 0.758, val_loss=0.456, val_auc = 0.612\n",
      "\n",
      "================================================================================2020-12-28 22:40:26\n",
      "[step=10] loss: 0.423, auc: 0.832\n",
      "[step=20] loss: 0.443, auc: 0.801\n",
      "[step=30] loss: 0.436, auc: 0.792\n",
      "[step=40] loss: 0.435, auc: 0.795\n",
      "\n",
      "EPOCH=8, loss=0.435, auc = 0.795, val_loss=0.462, val_auc = 0.645\n",
      "\n",
      "================================================================================2020-12-28 22:40:27\n",
      "[step=10] loss: 0.399, auc: 0.839\n",
      "[step=20] loss: 0.407, auc: 0.829\n",
      "[step=30] loss: 0.405, auc: 0.833\n",
      "[step=40] loss: 0.413, auc: 0.825\n",
      "\n",
      "EPOCH=9, loss=0.413, auc = 0.825, val_loss=0.488, val_auc = 0.638\n",
      "\n",
      "================================================================================2020-12-28 22:40:28\n",
      "[step=10] loss: 0.356, auc: 0.880\n",
      "[step=20] loss: 0.374, auc: 0.863\n",
      "[step=30] loss: 0.369, auc: 0.864\n",
      "[step=40] loss: 0.383, auc: 0.853\n",
      "\n",
      "EPOCH=10, loss=0.383, auc = 0.853, val_loss=0.496, val_auc = 0.681\n",
      "\n",
      "================================================================================2020-12-28 22:40:28\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "# 脚本训练风格\n",
    "epochs = 10\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=['epoch', 'loss', metric_name, 'val_loss', 'val_'+metric_name])\n",
    "\n",
    "print('start_training.........')\n",
    "nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "print('========'*8 + '%s' %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    \n",
    "    # 训练阶段\n",
    "    model.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = model(features);\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step=%d] loss: %.3f, \" + metric_name + \": %.3f\") % (step, loss_sum/step, metric_sum/step));\n",
    "    \n",
    "    # 验证阶段\n",
    "    model.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_val, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = model(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        \n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印日志\n",
    "    print((\"\\nEPOCH=%d, loss=%.3f, \" + metric_name + \" = %.3f, val_loss=%.3f, \" + \"val_\" + metric_name + \" = %.3f\") %info)\n",
    "    nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    print('\\n' + '=========='* 8 + '%s' %nowtime)\n",
    "    \n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:31.883860Z",
     "start_time": "2020-12-28T14:40:31.865950Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>loss</th>\n",
       "      <th>auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.598019</td>\n",
       "      <td>0.507588</td>\n",
       "      <td>0.481909</td>\n",
       "      <td>0.616401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.504629</td>\n",
       "      <td>0.660502</td>\n",
       "      <td>0.434602</td>\n",
       "      <td>0.658213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.488107</td>\n",
       "      <td>0.696503</td>\n",
       "      <td>0.446377</td>\n",
       "      <td>0.633484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.481433</td>\n",
       "      <td>0.713204</td>\n",
       "      <td>0.436295</td>\n",
       "      <td>0.601910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.474347</td>\n",
       "      <td>0.725325</td>\n",
       "      <td>0.443237</td>\n",
       "      <td>0.662293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.461928</td>\n",
       "      <td>0.749716</td>\n",
       "      <td>0.463261</td>\n",
       "      <td>0.629605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.457180</td>\n",
       "      <td>0.758042</td>\n",
       "      <td>0.456117</td>\n",
       "      <td>0.611799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.435261</td>\n",
       "      <td>0.795104</td>\n",
       "      <td>0.461597</td>\n",
       "      <td>0.645237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.412895</td>\n",
       "      <td>0.825446</td>\n",
       "      <td>0.488125</td>\n",
       "      <td>0.638105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.382952</td>\n",
       "      <td>0.852932</td>\n",
       "      <td>0.496133</td>\n",
       "      <td>0.681488</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch      loss       auc  val_loss   val_auc\n",
       "0    1.0  0.598019  0.507588  0.481909  0.616401\n",
       "1    2.0  0.504629  0.660502  0.434602  0.658213\n",
       "2    3.0  0.488107  0.696503  0.446377  0.633484\n",
       "3    4.0  0.481433  0.713204  0.436295  0.601910\n",
       "4    5.0  0.474347  0.725325  0.443237  0.662293\n",
       "5    6.0  0.461928  0.749716  0.463261  0.629605\n",
       "6    7.0  0.457180  0.758042  0.456117  0.611799\n",
       "7    8.0  0.435261  0.795104  0.461597  0.645237\n",
       "8    9.0  0.412895  0.825446  0.488125  0.638105\n",
       "9   10.0  0.382952  0.852932  0.496133  0.681488"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfhistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:40:38.183140Z",
     "start_time": "2020-12-28T14:40:37.895901Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU5fXA8e8JCYSAyL4IEsAiIIIgEXcBq4hIweKGBhWrUBcU14paNwrVqnWhP5fiAlpSlYILKoqiItXKElYFBBEDhEV2BAEhyfn98d6QSZhJJsnM3ElyPs9zn8zcuffOmUkyZ95dVBVjjDGmqAS/AzDGGBOfLEEYY4wJyhKEMcaYoCxBGGOMCcoShDHGmKAsQRhjjAnKEoSJCRGpJiJ7RKRlJI/1k4j8RkQi3k9cRM4RkayA+ytE5Mxwji3Dc70kIveW9fxirjtaRCZE+romthL9DsDEJxHZE3A3BfgVyPXu/1FVM0pzPVXNBWpH+tiqQFXbReI6InIdMFhVewZc+7pIXNtUTpYgTFCqeugD2vuGep2qzgh1vIgkqmpOLGIzxsSGVTGZMvGqEN4UkddFZDcwWEROFZHZIrJTRDaKyFgRSfKOTxQRFZFW3v2J3uMfishuEflaRFqX9ljv8fNFZKWI7BKRf4jIVyIyJETc4cT4RxFZJSI7RGRswLnVROQpEdkmIj8AfYp5f/4sIm8U2fesiDzp3b5ORJZ7r+cH79t9qGtli0hP73aKiPzLi20p0C3I8672rrtURPp7+zsB/wec6VXfbQ14bx8KOP9677VvE5F3RKRZOO9NSUTkQi+enSLymYi0C3jsXhHZICI/i8h3Aa/1FBFZ4O3/SUQeD/f5TISoqm22FbsBWcA5RfaNBg4Av8N90agJnAScjCuZtgFWAsO94xMBBVp59ycCW4E0IAl4E5hYhmMbA7uBAd5jtwMHgSEhXks4Mb4LHAm0Arbnv3ZgOLAUaAE0AGa5f6Ggz9MG2APUCrj2ZiDNu/877xgBzgb2AZ29x84BsgKulQ309G4/AcwE6gGpwLIix14KNPN+J1d4MTTxHrsOmFkkzonAQ97t3l6MXYBk4Dngs3DemyCvfzQwwbvdwYvjbO93dK/3vicBHYE1QFPv2NZAG+/2POBy7/YRwMl+/y9Utc1KEKY8vlTV91Q1T1X3qeo8VZ2jqjmquhoYB/Qo5vzJqpqpqgeBDNwHU2mP7QcsUtV3vceewiWToMKM8RFV3aWqWbgP4/znuhR4SlWzVXUb8Ggxz7Ma+BaXuADOBXaqaqb3+Huqulqdz4BPgaAN0UVcCoxW1R2qugZXKgh83kmqutH7nfwbl9zTwrguQDrwkqouUtX9wEigh4i0CDgm1HtTnEHAVFX9zPsdPQrUwSXqHFwy6uhVU/7ovXfgEn1bEWmgqrtVdU6Yr8NEiCUIUx7rAu+ISHsR+UBENonIz8AooGEx528KuL2X4humQx17VGAcqqq4b9xBhRljWM+F++ZbnH8Dl3u3r8Altvw4+onIHBHZLiI7cd/ei3uv8jUrLgYRGSIii72qnJ1A+zCvC+71Hbqeqv4M7ACaBxxTmt9ZqOvm4X5HzVV1BXAH7vew2auybOodeg1wHLBCROaKSN8wX4eJEEsQpjyKdvH8J+5b829UtQ7wAK4KJZo24qp8ABARofAHWlHliXEjcHTA/ZK64b4JnON9Ax+ASxiISE1gMvAIrvqnLvBxmHFsChWDiLQBngduABp41/0u4LoldcndgKu2yr/eEbiqrPVhxFWa6ybgfmfrAVR1oqqejqteqoZ7X1DVFao6CFeN+HdgiogklzMWUwqWIEwkHQHsAn4RkQ7AH2PwnO8DJ4rI70QkERgBNIpSjJOAW0WkuYg0AO4u7mBV/Qn4EhgPrFDV772HagDVgS1Aroj0A35bihjuFZG64saJDA94rDYuCWzB5crrcCWIfD8BLfIb5YN4HbhWRDqLSA3cB/V/VTVkiawUMfcXkZ7ec9+FazeaIyIdRKSX93z7vC0X9wKuFJGGXoljl/fa8soZiykFSxAmku4Arsb98/8T9w06qrwP4cuAJ4FtwDHAQty4jUjH+DyureAbXAPq5DDO+Teu0fnfATHvBG4D3sY19F6MS3TheBBXkskCPgReC7juEmAsMNc7pj0QWG//CfA98JOIBFYV5Z//Ea6q523v/Ja4dolyUdWluPf8eVzy6gP099ojagCP4dqNNuFKLH/2Tu0LLBfXS+4J4DJVPVDeeEz4xFXZGlM5iEg1XJXGxar6X7/jMaYisxKEqfBEpI+IHOlVU9yP6xkz1+ewjKnwLEGYyuAMYDWumqIPcKGqhqpiMsaEyaqYjDHGBGUlCGOMMUFVmsn6GjZsqK1atfI7DGOMqVDmz5+/VVWDdg2vNAmiVatWZGZm+h2GMcZUKCISckYAq2IyxhgTlCUIY4wxQVmCMMYYE1RU2yBEpA/wDG4CrpdU9bDpkUXkUuAh3Dwri1X1Cm//1RQMuR+tqq9GM1ZjTHw6ePAg2dnZ7N+/3+9QKrTk5GRatGhBUlKoqbgOF7UE4U158CxuHvxsYJ6ITFXVZQHHtAXuAU5X1R0i0tjbXx8350waLnHM987dEa14jTHxKTs7myOOOIJWrVrhJus1paWqbNu2jezsbFq3bl3yCZ5oVjF1B1Z5i6IcAN6gYPGUfEOBZ/M/+FV1s7f/POATVd3uPfYJxSzvWB4ZGdCqFSQkuJ8ZGSWdYYyJpf3799OgQQNLDuUgIjRo0KDUpbBoJojmFF7YJJvD5+k/FjhW3BrCs70qqXDPRUSGiUimiGRu2bKl1AFmZMCwYbBmDai6n8OGWZIwJt5Ycii/sryH0UwQwaIpOq9HItAW6IlbeeslEakb5rmo6jhVTVPVtEaNilsCILj77oO9ewvv27vX7TfGmKoumgkim8IrX7XATcNc9Jh3VfWgqv4IrMAljHDOLbe1a0u33xhjqpJoJoh5uAXHW4tIdbyFy4sc8w7QC0BEGuKqnFYD04HeIlJPROrh1uudHukAW4ZYMDLUfmNM/It0u+LOnTt57rnnSn1e37592blzZ6nPGzJkCJMnh7MWVfRFLUGoag5uOcTpwHJgkqouFZFRItLfO2w6sE1ElgGfA3ep6jZV3Q78BZdk5gGjvH0RNWYMpKQU3peS4vYbYyqeaLQrhkoQubm5xZ43bdo06tatW/YnjgNRHQehqtOAaUX2PRBwW4Hbva3oua8Ar0QzvnRvMcV773XVSjVqwLhxBfuNMfGnZ8/D9116Kdx4I9xzT/B2xREj3P/11q1w8cWFH585s/jnGzlyJD/88ANdunQhKSmJ2rVr06xZMxYtWsSyZcu48MILWbduHfv372fEiBEMGzYMKJgfbs+ePZx//vmcccYZ/O9//6N58+a8++671KxZs8TX+umnn3LnnXeSk5PDSSedxPPPP0+NGjUYOXIkU6dOJTExkd69e/PEE0/wn//8h4cffphq1apx5JFHMmvWrBKvX5IqP5I6Pd19y3j0Ufj1V0hL8zsiY0xZZWcH379tW9mv+eijj3LMMcewaNEiHn/8cebOncuYMWNYtswN6XrllVeYP38+mZmZjB07lm1Bnuz777/npptuYunSpdStW5cpU6aU+Lz79+9nyJAhvPnmm3zzzTfk5OTw/PPPs337dt5++22WLl3KkiVL+POf3XjiUaNGMX36dBYvXszUqUVr88um0szmWl5DhsCBA1C/vt+RGGOKU9w3/pYt3Re+olJT3c+GDUsuMZSke/fuhQabjR07lrfffhuAdevW8f3339OgQYNC57Ru3ZouXboA0K1bN7Kyskp8nhUrVtC6dWuOPfZYAK6++mqeffZZhg8fTnJyMtdddx0XXHAB/fr1A+D0009nyJAhXHrppQwcOLB8L9JT5UsQ+Zo0gfvvhzL0ljXGxIlYtCvWqlXr0O2ZM2cyY8YMvv76axYvXkzXrl2DDkarUaPGodvVqlUjJyenxOcJtdpnYmIic+fO5aKLLuKdd96hTx83fOyFF15g9OjRrFu3ji5dugQtyZSWJYgAubkweTJEoOrOGOOD9HTXjpiaCiLuZ3nbFY844gh2794d9LFdu3ZRr149UlJS+O6775g9e3bZn6iI9u3bk5WVxapVqwD417/+RY8ePdizZw+7du2ib9++PP300yxatAiAH374gZNPPplRo0bRsGFD1q1bV9zlw2JVTEXccQe0awcff+x3JMaYskhPj2xHkwYNGnD66adz/PHHU7NmTZo0aXLosT59+vDCCy/QuXNn2rVrxymnnBKx501OTmb8+PFccsklhxqpr7/+erZv386AAQPYv38/qspTTz0FwF133cX333+PqvLb3/6WE044odwxSKhiTEWTlpamkVhR7uGH4aGHYPVqKMWcVsaYKFm+fDkdOnTwO4xKIdh7KSLzVTVo9xyrYiriD39wA2xeftnvSIwxxl+WIIo4+mjo0wfGj4cw2pGMMaZMbrrpJrp06VJoGz9+vN9hFWJtEEEMHQp33+0Gz7Vp43c0xpjK6Nlnn/U7hBJZCSKI/v3hu+8sORhjqjZLEEEkJLgucvv2QYjebcYYU+lZgghh505o3hyeecbvSIwxxh+WIEKoWxe6dnW9mfLy/I7GGGNizxJEMYYOhawsmDHD70iMMWHzeaH52rVrh3wsKyuL448/PobRlI8liGL8/vfQoAG8+KLfkRhjwmILzUeUdXMtRo0acNVV8H//5+aRb9jQ74iMqeJuvRW8uYeCmj3bzdsfaO9euPba0N/0unSBp58Oecm7776b1NRUbrzxRgAeeughRIRZs2axY8cODh48yOjRoxkwYECpXsr+/fu54YYbyMzMJDExkSeffJJevXqxdOlSrrnmGg4cOEBeXh5TpkzhqKOO4tJLLyU7O5vc3Fzuv/9+LrvsslI9X1lYgijBiBEwaJArSRhj4lzR5FDS/jAMGjSIW2+99VCCmDRpEh999BG33XYbderUYevWrZxyyin0798fEQn7uvnjIL755hu+++47evfuzcqVK3nhhRcYMWIE6enpHDhwgNzcXKZNm8ZRRx3FBx98ALhJAmPBEkQJUlML5pI3xvismG/6gGtzCLUgRBkXgujatSubN29mw4YNbNmyhXr16tGsWTNuu+02Zs2aRUJCAuvXr+enn36iadOmYV/3yy+/5OabbwbczK2pqamsXLmSU089lTFjxpCdnc3AgQNp27YtnTp14s477+Tuu++mX79+nHnmmWV6LaVlbRBh2LzZNVh/9ZXfkRhjihWlBSEuvvhiJk+ezJtvvsmgQYPIyMhgy5YtzJ8/n0WLFtGkSZOg60AUJ9REqVdccQVTp06lZs2anHfeeXz22Wcce+yxzJ8/n06dOnHPPfcwatSocr2ecFkJIgy1asGkSbB/P5x+ut/RGGNCyp/n+7773Fw5LVu65FDO+b8HDRrE0KFD2bp1K1988QWTJk2icePGJCUl8fnnn7MmWKmlBGeddRYZGRmcffbZrFy5krVr19KuXTtWr15NmzZtuOWWW1i9ejVLliyhffv21K9fn8GDB1O7dm0mTJhQrtcTLksQYahVy/19jR8PY8dCvXp+R2SMCSnSC0IAHTt2ZPfu3TRv3pxmzZqRnp7O7373O9LS0ujSpQvt27cv9TVvvPFGrr/+ejp16kRiYiITJkygRo0avPnmm0ycOJGkpCSaNm3KAw88wLx587jrrrtISEggKSmJ559/PqKvLxRbDyJMCxfCiSe6BOFVGxpjYsDWg4gcWw8iSrp2hW7dXE+5SpJTjTGmWFbFVAq33gpz5rgec8nJfkdjjIlX33zzDVdeeWWhfTVq1GDOnDk+RVQ2liBKYfBgtxljYktVSzXGwG+dOnViUXED+nxQluYEq2IqJVWYNcumATcmVpKTk9m2bVuZPuCMo6ps27aN5FJWfVgJopTmz4cePWDcODc2whgTXS1atCA7O5stW7b4HUqFlpycTIsWLUp1jvViKiVV6NwZataEuXOj/nTGGBNV1ospgkTguutg3jxYvNjvaIwxJnosQZTBlVe6mV5tGnBjTGVmCaIM6teHiy6Cjz+21eaMMZWXNVKX0VNPuWVJEyzFGmMqKUsQZdS4sfup6toljDGmsrHvv+Xw9ddwzDHw3Xd+R2KMMZFnCaIc2rSBdevgpZf8jsQYYyLPEkQ5NGkCAwbAq6+Wa0VDY4yJS5Ygyum662DrVnj3Xb8jMcaYyIpqghCRPiKyQkRWicjIII8PEZEtIrLI264LeCw3YP/UaMZZHuee6xatsjERxpjKJmq9mESkGvAscC6QDcwTkamquqzIoW+q6vAgl9inql2iFV+kVKsGjz9++DK4xhhT0UWzm2t3YJWqrgYQkTeAAUDRBFHhXXqp3xEYY0zkRbOKqTmwLuB+trevqItEZImITBaRowP2J4tIpojMFpELgz2BiAzzjsn0e6bHdetg9GjIyfE1DGOMiZhoJohgw8eKTh37HtBKVTsDM4BXAx5r6c0weAXwtIgcc9jFVMepapqqpjVq1ChScZfJggVw//0wbZqvYRhjTMREM0FkA4ElghbAhsADVHWbquZ3EH0R6Bbw2Abv52pgJtA1irGWW9++0LSpNVYbYyqPaCaIeUBbEWktItWBQUCh3kgi0izgbn9gube/nojU8G43BE4nztsukpLgmmtcCSI72+9ojDGm/KKWIFQ1BxgOTMd98E9S1aUiMkpE+nuH3SIiS0VkMXALMMTb3wHI9PZ/DjwapPdT3Ln2Wje76/jxfkdijDHlZyvKRVjfvtCunZvt1Rhj4l1xK8rZbK4R9v77NgW4MaZysI+yCMtPDhs3+huHMcaUlyWIKPjHPyA1FTZv9jsSY4wpO0sQUXDOOXDwoJvl1RhjKipLEFHQoQOcfrpbJ6KS9AEwxlRBliCiZOhQWLkSZs3yOxJjjCkbSxBRcsklUKcOvPKK35EYY0zZWDfXKElJgalToXNnvyMxxpiysQQRRT16+B2BMcaUnVUxRdnUqTB4sDVWG2MqHksQUbZpE2RkwNy5fkdijDGlYwkiygYNcu0RNg24MaaisQQRZXXquCTxxhuwe7ff0RhjTPgsQcTA0KHwyy/w+ut+R2KMMeGzBBEDJ58M6enQrFnJxxpjTLywbq4xIAITJ/odhTHGlI6VIGJoxw6YOdPvKIwxJjyWIGLo1lvhwgth716/IzHGmJJZgoiha66BXbtg8mS/IzHGmJJZgoihHj2gbVsbE2GMqRgsQcSQCFx3HXz5JSxf7nc0xhhTPEsQMXb11ZCUBDNm+B2JMcYUz7q5xliTJrBmjY2JMMbEPytB+CA/OeTl+RuHMcYUxxKET269FX73O7+jMMaY0CxB+KR+fZg2DVav9jsSY4wJzhKET/7wB0hIgJdf9jsSY4wJzhKET1q0gPPPh/HjISfH72iMMeZwliB8NHQobNwIH3zgdyTGGHM4SxA+6tsXxoyBbt38jsQYYw5n4yB8lJQE997rdxTGGBOclSDiwHvvwaRJfkdhjDGFWYKIA888A3/6kw2cM8bEF0sQcaBDBzf9RmIitGoFGRl+R2SMMZYgfJeRAa+84m6rukQxbJglCWOM/yxB+Oy++w5fYW7vXrffGGP8ZAnCZ2vXht6/f39sYzHGmEBRTRAi0kdEVojIKhEZGeTxISKyRUQWedt1AY9dLSLfe9vV0YzTTy1bBt9fowakpsKoUbBlS2xjMsYYiGKCEJFqwLPA+cBxwOUiclyQQ99U1S7e9pJ3bn3gQeBkoDvwoIjUi1asfhozBlJSCu9LSYHbb4eTToIHH3RJ5PrrYcUKf2I0xlRN0SxBdAdWqepqVT0AvAEMCPPc84BPVHW7qu4APgH6RClOX6Wnw7hxrrQg4n6OG+cSx/vvw7JlcOWVMGECTJ7szsnLcw3axhgTTdFMEM2BdQH3s719RV0kIktEZLKIHF2ac0VkmIhkikjmlgpcD5OeDllZ7oM/K8vdz9ehg0sYa9fC8OFu36RJrnTx+utw8KAfERtjqoJoJggJsq/o9973gFaq2hmYAbxainNR1XGqmqaqaY0aNSpXsPGucWM48kh3u2ZN2LMHrrgCjjkG/v532LXL3/iMMT7IyHCDpxISojKIKpoJIhs4OuB+C2BD4AGquk1Vf/Xuvgh0C/fcqmzAAFf19N57LkHceSece67fURljYiojww2aWrMmaoOowkoQIjJCROqI87KILBCR3iWcNg9oKyKtRaQ6MAiYWuS6zQLu9geWe7enA71FpJ7XON3b22c8CQnQrx98/jlkZsKjj7r9e/bANdfAvHn+xmeMiRJVWLfOrVsc5UFU4ZYg/qCqP+M+qBsB1wCPFneCquYAw3Ef7MuBSaq6VERGiUh/77BbRGSpiCwGbgGGeOduB/6CSzLzgFHePhNEt25w9tnu9uLF8NZb0L07nHUWvPuuzfFkTIX2yy8waxY89hgMHOhWG2vZErZuDX58qMFVZSAaRncYEVmiqp1F5Blgpqq+LSILVbVrxCIpp7S0NM3MzPQ7jLjw889uKdOnn3Z/K23bwuzZbh1sY0wcy8uDlSthzhz3Tzt7NnzzDeTmusePOQZOOcVtY8bApk2HXyM11fV2CZOIzFfVtGCPhbsexHwR+RhoDdwjIkcA9r00TtWpA7fdBjffDFOmwMyZBcnhnXfc31bTpr6GaIwB2L7dJYP8hDBnDuzc6R6rU8dVBdxzj/un7d4dAjvj1Kvn2hwCq5lSUlziiJBwSxAJQBdgtaru9AaytVDVJRGLpJysBFGyn392iSE3FwYPdoPxOnb0OypjqoiDB11pIL9kMGeOKy2Aa1Q8/ng4+eSCEkL79m5/cTIyXJvD2rWu2mnMmML95MNQXAkCVS1xA04Hanm3BwNPAqnhnBurrVu3bmpKtnKl6g03qNasqQqqffqofvute2ziRNXUVFUR93PiRD8jNSaOhfPPsm6d6uTJqnfeqXrGGQX/dKDapInqgAGqjzyi+tlnqj//HOtXcAiQqSE+V8NugwBOADoD/wJeBgaqao9SpaooshJE6WzbBi+8AM89B1984b7MDB0K+/YVHJOS4gbplfILiTGVW3730sCqnZo1XX/zOnUKSgjr17vHqleHE08sKBmcfHLB1AlxoLgSRLgJYoGqnigiDwDrVfXl/H2RDrasLEGUTW4uVKvmxtisWXP44y1bBt9vTJUV6p8lX5s2hauKTjjBzb4ZpyLRSL1bRO4BrgTO9CbiS4pUgMY/1aq5n8VNO57vrbfc/8Zxx0FyctRDMyY+hfpnEXG9iho3jm08URTuOIjLgF9x4yE24eZFejxqUZmYCzXteMOG7uevv8Jll7kxF7Vru/a0K65wEwpCfsVqbGI1xjebNoUuDbRsWamSA4SZILykkAEcKSL9gP2q+lpUIzMxFWra8aefdreTkuDbb91EgSNHulL0V1/Bd9+5x9evd8nk7LPdAM/x42H+fFv0yFQiH37oqotycly7QqAIdy+NG6FarwM34FJgDW4yvdeAH4GLwzk3VluZezFZ151DyvJW5OW5n1lZqkOHqnbvXrizRkaGe3zlStW//lX1gw9c54788yIVhzFRs3+/6ogR7g+6UyfX7a8S/ZFSTC+mcBPEYqBxwP1GwOJwzo3VVqYEMXGiakpKwacZuPsV+JcdD3JyVFesUP3Pf1Q3bHD7Jk4s/DbXr6/as6fq99+7x3fuVN27134lJs4sW6Z6wgnuD/Hmm1X37fM7oogrLkGE24vpG1XtFHA/wUsQnYo5LabK1IspVG+EUg5VN+HZudNVUy1eDEuWuO3996FBA7e06sMPu3FBOTmHn2u/EhNTqvDii66+tFYtV2far5/fUUVFJHoxfSQi04HXvfuXAdMiEZyvwum6YyKmbl044wy3FXXuua7L7ahRwc9dswbuvRe6dnVbmzYlDzI1pky2b3eDgt56C845B157DZo1K/m8SiisEgSAiFyEG1EtwCxVfTuagZWWlSAqh1C/klq1XE+q/NJFnTruC13+1Pdr18JRR0FiuF95jAnmiy/cPDSbNsFf/wp33FHpv4kUV4II+5Wr6hRVvV1Vb4u35FBmwbruJCZWzt4IFUSo3lT//Kdb6yIz05X809NdHs935pmu+2337vDHP7pR4kuXxjZ2U4EdPAj33w+9erlR0V9/DXfdVemTQ0mK/b4lIrsJstQnrhShqlonKlHFSv4cEvmTXR1xhJvRLlgluImJor+SovOPdevmtkCq8MgjsGABLFzouuKOGwc33gjPPut+nUOHQufOrnqqSxdX3WUMAD/+6Ab1zJ7tVtsaO9Z92zDh9WKqCFtEJus7eFC1Vy/V5GTVhQvLfz3ji7w81R9/dF1vVVXXrlU96qjCPaPatFGdNMk9vm+f6saNh1+nEvVkNKFkZKjWqeO211/3OxpfUEwvJquxDZSYCK+/7r6iXnSRq8+oV8/vqEwpibi2jHxHH+0G8v30kythLFzoShv5U+t/+aVrJG/a1M2p1rUr7N7tqrLyJy/MX+4XbPLCSmH3bhg+3DVAn3aaa8wK/KMxQCkaqeNdRCfr+/pr6NEDzjvPrdlZxeshK7u1a12HlfzksWxZwQJeRVn/hUpg3jy4/HJXtfTnP7u2hyrcuyEijdRVyqmnwpNPuk76jzzidzQmylq2dN3dX33Vjc3YvTv0TMxr1sAbb7gGc1PB5OXB3/7mSgwHDrilFh9+uEonh5JYggjlpptcw9X998Mnn/gdjYmhmjVDT16YkOC+fHbpUjA5YZ4tvhv/Nmxw9YgjR8KFF7rRmmee6XdUcc8SRCgiritMx47uE8EGz1UpobrbTpjguso/8oj7E8nLgw4d3J/I228XXnDJxImpU10Xttmz4aWXXDc3a1sMiyWI4tSqBVOmuD7SF1/sRmqZKiE93X0/yF/4KzXV3b/ySjjrLLjkEnfcnj2u6/yMGTBwoJvtefBg1whufLZvn6sJGDDA9VSYPx+uvTZuVnKrCCxBlOTYY93XxnnzYMQIv6MxMZSe7hqk8/Lcz2C9l+rUcYPyNm6Ejz+GQYPcrNA//eQeX70apk1zVd4mhr791o2afO45uP12VyTcS/wAABbvSURBVHpo397vqCocSxDh+P3v4U9/csN5X33V72hMHEpMdFXcL77oZmno3dvtnzABLrgAmjSBP/wBpk93BVITJapudGRaGmze7LL13/8e10t+xjPr5hqunBz3X//1127r0iV6z2UqjQMHXB+HN990PaZ//tnVdvz4Y8FyryZCtm51Wfi99+D8890MrE2a+B1V3LNurpGQP4iufn03iG7HDr8jMhVA9equBPHaa67a6d133RQ/+cnhvPPg+uvh889Dj704NIgrIcH9zJ+h0BSYMcM1RE+fDk895bqoW3IoN0sQpdGkCfznP65H01VXWf9GUyrJydC/P9x8s7v/66+uM82//uWWam3e3A3uXbw44KSMDHL+MMwNwFCFNWvcfUsSzoEDcPfdrnR/5JEwZ44b1GKDWyPC3sXSOu20gm8oNojOlEONGm7Q3ZYtruflmWfCK68U9IDauhV23nQfiQf2Fjov8cBe9oy414eI40Bgaap5c9fw/NhjbjbGzEyr+o0wa4MoC1XXl/H1112R9txzY/O8ptLbs8d99qWkwD8f28Wwu+sSrFOmAtKrV8HkUSee6HrcVeaGjYwMNyHW3sIJk1tugWee8SemSqC4NghLEGX1yy9wyimuf+OCBaGH3hpTWr/+Ci+8QN6ov5CwfVvQQ3ZTmz1HH0eD9YupnufG5+TUSOFA+xNIOePEgsTRsaNrCKloVN3/1sqVBdtzzwUfiWgTZJWLJYhoWbkSTjoJ2rWD//7XutKZ8snLc92d7rvPdXM691wem9uDm3b9lVoUfGv+hRTuaTCOJrelM/n1g1Rf/R3t9y2gKws5OWkBp9ZYeGiyqJyEJDY16sTu33SFridyZK8TOapP58OHiftl5074/vuCJLBiRcHtX34pOC45GfbvD36N/CHtpkwsQUTT22+7IbT5y5gZUxaffebG2syfDyec4OrVe/cmIwNmXJPBgwfvoyVrWUtLHk4awznj0w8N3FN17RU//gi7dsG5v82DVat4efhCcuYt4JhdC+iiC2mIVxpJSID27Zl98ERWH3kiBzp2pXr3LrQ4vi5t24ZefjkjI/RCTsX69VdYtapwaSB/27y54Lj8XlrHHluwtWvnfrZo4RYityWCI84SRLTdfbf7h54wAa6+2p8YTMW0ZIn7+/noI/dBN3q0myQyoBdOmT+YPbm5sD5b2TBnHbVWLKBTjlsQY9uMBTTYv+HQcas4hs1HdeW04a6KatjzXUlo2pjWrSE7G35+IYOHc0IkqtxcWLcueBLIyiqY2RBcb8DAD//8rU2b4kvhwdogUlLcHCi2SEeZWYKINhtEZ0pr7Vp44AE3QKJuXZcBbrrJVaXE0k8/cWDuQn7+fAE58xZQd/VCkjesPvTwhoTmZOadyEESuYBpJFMwH9kBElmW1IUux+5zJYTAucpq1z48ARx7LLRt67qjllV5s6U5jCWIWPjpJ9cwmJxsK9GZ0HbsgEcfLeh1c8stcM898fX3smMHLFp0aOm93MwFJKxYHrQ31QESqd6/7+GJoGlTmxSvgrAEESv/+59bia5PH1uJzhS2f7+bI2jMGNcwe9VVMGpUhen9licJJHD4Z0UewqzP8+jZM/YxmciwqTZi5bTTbCU6U1heHkyc6Kpb7rwTTj7ZfTufMKHCJAeAvQ2Cx7o+oSW9ern1MLKzYxyUibqoJggR6SMiK0RklYiMLOa4i0VERSTNu99KRPaJyCJvqzjdg4YPt5XojPPxx9Ctm1tEolEj+PRTN7to585+R1ZqtZ8ZQ071wl1jc6qn0OSlMTz4oOvMZ534KiFVjcoGVAN+ANoA1YHFwHFBjjsCmAXMBtK8fa2Ab0vzfN26ddO4sWeP6vHHqzZooLpmjd/RmFhbsED1nHNUQbV1a9V//1s1N9fvqMpv4kTV1FRVEfdz4sRDD/3wg+ru3e72F1+oTp/uS4SmDIBMDfG5Gs0SRHdglaquVtUDwBvAgCDH/QV4DAgxCqYCspXoqqasLDcFy4knugbep5+G5ctd/UtlaI8qZgWlNm1cxyWAv/3NzVJ70UXBhy2YiiOaf7XNgXUB97O9fYeISFfgaFV9P8j5rUVkoYh8ISJBVxcXkWEikikimVu2bIlY4BFhK9FVHdu2wR13uHaGKVNcr6QffnC/9yo4uv6tt1xb/IcfuvW6R48OPQjaxLdoJohQc4y5B0USgKeAO4IctxFoqapdgduBf4tIncMupjpOVdNUNa1Ro0YRCjuCKtpKdLbuQOns2+cGSB5zjCstDB7spo3461/L19e/gqtRA+69F777zq2Fcf/9brZaUwGFqnsq7wacCkwPuH8PcE/A/SOBrUCWt+0HNuC1QxS51sxg+wO3uGqDCHTwoGqvXqrJyaoLF/odTWgTJ6qmpLh68/wtJaVQPbPx5OSojh+v2qKFe5/69VP95hu/o4pbX37p3jJV1Q8/dO0VJn5QTBtENBNEIrAaaE1BI3XHYo4/lASARkA173YbYD1Qv7jni9sEoaq6aZPqUUeptmmjun2739Ec7sAB1caNCyeH/K1xY9VffvE7wviQl6c6bZpqp07uvTnpJNWZM/2OqsI4eNC1bdeoofrAA/ZnFS+KSxBRq2JS1RxgODAdWA5MUtWlIjJKRPqXcPpZwBIRWQxMBq5X1e3RijXq4nElur174Z13XDyNGxeeNC3Q5s1ulO8557jqlEWL4iP+WAiscmvWzE2d3bevq1qaNMmtXtajh99RVhiJifDVV25uy1Gj4Ljj3J+gVo6xupVTqMxR0ba4LkHkGzvWffMcPdqf59+5UzUjQ/Wiiwqqk+rVU736atVGjUKXIO64o+BbM6g2aaI6eLDqa6+pbtzoz2uJtmBVbqB61VWqv/7qd3QV3syZric4qM6d63c0VRt+VDHFeqsQCSIvT/Xyy10/8o8/js1z/vST6osvqp5/vmpSkvuVN2umeuONqjNmuOol1fDaINavV331VdX09MIJpXNn1TvvdK9p797YvK5I+/lnV1n+j3+oXnutavXqwRNmaqrfkVYaBw6oTp1acH/atIKxFCZ2iksQNhdTrP3yi5tuYdOm6K1Et26dG9r61ltuIaO8PNdRfeBAt518cvB++aWZKTMvDxYvdqOFP/4YvvzSLSCfnOyqXXr3dlvHjvE1aZt6K5UtWlSwLVzoZiPN16CB67oajC1OExWbNrnZzhs3hr//HS65JL7+bCozm6wv3qxcCWlpbsH1SK1Et2KFSwhvv+3GXgAcf3xBUujcObr/cb/8ArNmFSSMZcvc/mbNCpLFOee4T4BYyc113U4XLiycEALbW9q0cUtzdulSsDVvDq1b2+I0Mfa//7mZahYuhF694B//cN8vTHRZgohH5V2JTtV92L31ltvyP5C7d3fX/f3v3WA9v6xb5+ai+vhj93O718ega1c3zLZ3bze5YaQGku3dC998U7hUsGRJwRrGSUkuYeYnga5dXdIMNV7BFqfxRW6ue4vvu88Nrlu3zhXoTPQUlyB8bzuI1FYh2iCK+tOfXL32hAnhHZ+b6+rJb79dtVUrd25CgmrPnq4BfO3a6MZbVjk5qvPmqY4Zo3rWWaqJiQVtHH37qj79tOqyZa6NRrXYOX9UVXXzZtfe8be/uTadDh3c+5DfTnDkkao9eqjeeqt7bxctKlvDcklxmKjZskV18uSC+zNnFvx5mMjC2iDiVP5KdP/9r5vtc9Omw+v+Dx6EmTNdKeGdd9wx1au76pqBA6F/f3duRbJ7t3tN+dVRK1e6/S1auFHJX3/t2jPyVa8O55/v3otFi2BDwTKZtGxZuHqoa1dXDWQV2JXGV1/BGWfAmWe6P4N//tMWlIskq2KKZ8895ypeA38PNWvC9de7htKpU90CM7VquT74v/+9+1mZpnLIyiqojpoyJXTH+Pwqovw2gxNOsPqHKiAvD155BW691TV1BbJav/KzBBHPWrUKPeVlvXquhDBwIJx7rksclV1CQvAEYb2Hqryjjw6+KJH1Gyif4hJEYqyDMUWsXRt8v4hb5zopKbbx+K1ly+AJswKtvmaiY/364PtD/QuZ8qsEk9RXcKE++Fq2rHrJAVylckrhlctISXH7TZUW6l9F1a3munt3bOOpCixB+M0+EAtLT3eVyvkNzampVslsgOD/KjVrQs+ebnBdhw5uyrNKUmseFyxB+M0+EA9XzMplpuoK9q/y4ovw+edukF2jRnDppXDDDX5HWnlYI7UxplLIyYHnn3cTFJx7bsEYyarQt6M8imukthKEMaZSSEyEm292yQHcUqcdO8IHH/gbV0VmCcIYUymdc46byaVfPzd8KFRvchOaJQhjTKXUq5ebcPjRR90YzA4d4I03/I6qYrEEYYyptKpXh7vvdnNZ9u3rBuODa68wJbOBcsaYSi81FSZPLrh/1VWuJ9QTT7gZ6U1wVoIwxlQpqtC2rUsY7dvD2LFWogjFEoQxpkoRgYcfhm+/hVNOgREj4KSTYPlyvyOLP5YgjDFVUtu28NFHbvR1Tg7Ur+93RPHHEoQxpsoSgYsvdosPNmniBu/37w8vvWSTB4MlCGOMObS+1I4dbvmVoUPh9NPd+lRVmSUIY4zxNGgAX3wBr74KP/wA3bq5Noo9e/yOzB+WIIwxJoCI6wa7YoVb2PHDD6FaNb+j8oclCGOMCaJePXj2WVfNVLMm7N3rEsfy5ZCR4RaDTEhwPzMy/I42OmygnDHGFCN/DYpvv4X33nPJICGhYOzEmjUwbJi7XdlmprcShDHGhKF7d1ftVLPm4QPr9u6F++7zJ65osgRhjDFhatzYJYNgKuPa2JYgjDGmFEKtjd2iRWzjiAVLEMYYUwrB1sauVs1VO61c6U9M0WIJwhhjSiHY2tgjR8LBg66d4qOP/I4wcixBGGNMKaWnQ1aWm44jK8stb5qZ6bq8XnABPP64mzW2orMEYYwxEZCaCl99BRddBE8/7abtqOhsHIQxxkRIrVrw5puwfr2bHTY3F7ZsgaZN/Y6sbKwEYYwxESRS0KPpgQegSxdXsqiILEEYY0yUpKfDEUdAr15uCvGKJqoJQkT6iMgKEVklIiOLOe5iEVERSQvYd4933goROS+acRpjTDQcdxzMnesSxNChcPPNrrdTRRG1BCEi1YBngfOB44DLReS4IMcdAdwCzAnYdxwwCOgI9AGe865njDEVSr168MEHcMcdrhSxYoXfEYUvmiWI7sAqVV2tqgeAN4ABQY77C/AYsD9g3wDgDVX9VVV/BFZ51zPGmAonMRGeeMLNBHv88W7fpk3+xhSOaCaI5sC6gPvZ3r5DRKQrcLSqvl/ac73zh4lIpohkbtmyJTJRG2NMlLRq5X6+8Qb85jcwZYqv4ZQomglCguw7NHRERBKAp4A7SnvuoR2q41Q1TVXTGjVqVOZAjTEmlnr0gE6d3HrYDzwQv+tfRzNBZANHB9xvAWwIuH8EcDwwU0SygFOAqV5DdUnnGmNMhdWsGXz+OQwZAn/5CwwcCLt3+x3V4aKZIOYBbUWktYhUxzU6T81/UFV3qWpDVW2lqq2A2UB/Vc30jhskIjVEpDXQFpgbxViNMSamkpPhlVfgmWfg/fdhxgy/Izpc1EZSq2qOiAwHpgPVgFdUdamIjAIyVXVqMecuFZFJwDIgB7hJVXOjFasxxvhBBG65xc3fdMwxbt/mzW7diXggWhlmlALS0tI0MzPT7zCMMabM5s+Hs85yU4qPGOESSLSJyHxVTQv2mI2kNsaYONGuHZx3Htx2G1xzDezfX/I50WQJwhhj4kTt2jB5Mjz0ELz6KvTsCRt87J5jCcIYY+JIQgI8+KAbI/Htt64h2y823bcxxsShgQOhc2do3drd37YNGjSIbQxWgjDGmDj1m9+49a43bXID626/3a19HSuWIIwxJs41aOBGXT/1FPTtC9u3x+Z5LUEYY0ycS0qCsWPdbLAzZ0L37rB0afSf1xKEMcZUENde6xLEnj0wahRkZLgJABMS3M+MjMg+nzVSG2NMBXLaaZCZCR99BMOGwd69bv+aNe4+uJXsIsFKEMYYU8G0aAGjRxckh3x798J990XueSxBGGNMBbR2ben2l4UlCGOMqYBatizd/rKwBGGMMRXQmDGQklJ4X0qK2x8pliCMMaYCSk+HceMgNdXN+pqa6u5HqoEarBeTMcZUWOnpkU0IRVkJwhhjTFCWIIwxxgRlCcIYY0xQliCMMcYEZQnCGGNMUKKqfscQESKyBVjjdxzl1BDY6ncQccTej8Ls/Shg70Vh5Xk/UlW1UbAHKk2CqAxEJFNV0/yOI17Y+1GYvR8F7L0oLFrvh1UxGWOMCcoShDHGmKAsQcSXcX4HEGfs/SjM3o8C9l4UFpX3w9ogjDHGBGUlCGOMMUFZgjDGGBOUJYg4ICJHi8jnIrJcRJaKyAi/Y/KbiFQTkYUi8r7fsfhNROqKyGQR+c77GznV75j8JCK3ef8n34rI6yKS7HdMsSQir4jIZhH5NmBffRH5RES+937Wi8RzWYKIDznAHaraATgFuElEjvM5Jr+NAJb7HUSceAb4SFXbAydQhd8XEWkO3AKkqerxQDVgkL9RxdwEoE+RfSOBT1W1LfCpd7/cLEHEAVXdqKoLvNu7cR8Azf2Nyj8i0gK4AHjJ71j8JiJ1gLOAlwFU9YCq7vQ3Kt8lAjVFJBFIATb4HE9MqeosYHuR3QOAV73brwIXRuK5LEHEGRFpBXQF5vgbia+eBv4E5PkdSBxoA2wBxntVbi+JSC2/g/KLqq4HngDWAhuBXar6sb9RxYUmqroR3BdOoHEkLmoJIo6ISG1gCnCrqv7sdzx+EJF+wGZVne93LHEiETgReF5VuwK/EKHqg4rIq1sfALQGjgJqichgf6OqvCxBxAkRScIlhwxVfcvveHx0OtBfRLKAN4CzRWSivyH5KhvIVtX8EuVkXMKoqs4BflTVLap6EHgLOM3nmOLBTyLSDMD7uTkSF7UEEQdERHB1zMtV9Um/4/GTqt6jqi1UtRWu8fEzVa2y3xBVdROwTkTaebt+CyzzMSS/rQVOEZEU7//mt1ThRvsAU4GrvdtXA+9G4qKJkbiIKbfTgSuBb0RkkbfvXlWd5mNMJn7cDGSISHVgNXCNz/H4RlXniMhkYAGu999Cqti0GyLyOtATaCgi2cCDwKPAJBG5FpdEL4nIc9lUG8YYY4KxKiZjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE5QlCGOMMUFZgjCmBCKSKyKLAraIjWQWkVaBs3IaE09sHIQxJdunql38DsKYWLMShDFlJCJZIvI3EZnrbb/x9qeKyKcissT72dLb30RE3haRxd6WP0VENRF50Vvj4GMRqekdf4uILPOu84ZPL9NUYZYgjClZzSJVTJcFPPazqnYH/g83Cy3e7ddUtTOQAYz19o8FvlDVE3DzKS319rcFnlXVjsBO4CJv/0igq3ed66P14owJxUZSG1MCEdmjqrWD7M8CzlbV1d5ki5tUtYGIbAWaqepBb/9GVW0oIluAFqr6a8A1WgGfeAu9ICJ3A0mqOlpEPgL2AO8A76jqnii/VGMKsRKEMeWjIW6HOiaYXwNu51LQNngB8CzQDZjvLZBjTMxYgjCmfC4L+Pm1d/t/FCyDmQ586d3+FLgBDq25XSfURUUkAThaVT/HLZ5UFzisFGNMNNk3EmNKVjNgll1w60Pnd3WtISJzcF+2Lvf23QK8IiJ34VaDy599dQQwzptxMxeXLDaGeM5qwEQRORIQ4ClbatTEmrVBGFNGXhtEmqpu9TsWY6LBqpiMMcYEZSUIY4wxQVkJwhhjTFCWIIwxxgRlCcIYY0xQliCMMcYEZQnCGGNMUP8PYrm/3CgCUOwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU9dXA8e8h7DsCKoIkKKiIbBIBtbXuorUiFRVEX6EqdV9bpeJCVVpt1VYrLqiolSgqKqJFrPuCigQBFVxYChgBCQEEZAvJef84N2QIMyEhc3Mnyfk8zzwzc7c5mSRz5reLquKcc86VVCvqAJxzzqUmTxDOOefi8gThnHMuLk8Qzjnn4vIE4ZxzLi5PEM455+LyBOEqjYikicgGEWmfzGOjJCIdRSTpfcVF5HgRWRzz/FsR+WVZjt2N13pMRG7c3fNd9VU76gBc6hKRDTFPGwJbgILg+e9VNas811PVAqBxso+tCVT1wGRcR0QuBM5V1aNjrn1hMq7tqh9PEC4hVd3+AR18Q71QVd9KdLyI1FbVbZURm3MufF7F5HabiNwhIs+JyLMish44V0QOF5FPRWStiCwXkftFpE5wfG0RURHJCJ6PD/a/LiLrReQTEelQ3mOD/SeLyHci8pOI/EtEponI0ARxlyXG34vIAhFZIyL3x5ybJiL/EJE8EVkI9Cvl/blJRCaU2DZGRO4NHl8oIl8HP8/C4Nt9omvliMjRweOGIvJ0ENtcoFec110UXHeuiJwWbO8KPAD8Mqi+WxXz3o6KOf/i4GfPE5FJItKmLO9NOd/nnarmROSj2N9Z8DrfBD/HVyLSPdFruZCoqt/8tssbsBg4vsS2O4CtwG+wLxsNgMOAPljpdD/gO+Dy4PjagAIZwfPxwCogE6gDPAeM341j9wTWA/2DfdcC+cDQBD9LWWJ8BWgGZACri3524HJgLtAOaAl8YP9GcV9nP2AD0Cjm2iuBzOD5b4JjBDgW2AR0C/YdDyyOuVYOcHTw+G7gPaAFkA7MK3HsWUCb4HdyThDDXsG+C4H3SsQ5HhgVPD4xiLEHUB94EHinLO9NOd/njiXfN+Cjot8ZMBj4Hkt+AhwA7Bv1/0FNu3kJwlXUR6r6qqoWquomVZ2hqtNVdZuqLgLGAr8q5fyJqpqtqvlAFvbBVN5jTwVmq+orwb5/YMkkrjLG+FdV/UlVF2MfxkWvdRbwD1XNUdU84M5SXmcR8BWWuABOANaqanaw/1VVXaTmHeBtIG5DdAlnAXeo6hpVXYKVCmJf93lVXR78Tp7BkntmGa4LMAR4TFVnq+pmYATwKxFpF3NMovdmB7vxtxDrQuBOVZ0ZvD/fqer3ZTzXJYknCFdRO/zTishBIvIfEVkhIuuA24BWpZy/IubxRkpvmE507D6xcaiqYt+44ypjjGV6LWBJKfECPIN9Gwb7Nr+9YV9EThWR6SKyWkTWYt/eS3uvirQpLQYRGSoic4KqnbXAQWW8LtjPt/16qroOWAO0jTmmTL+z3fhbiLUvsLCMx7qQeIJwFVWyi+cj2LfmjqraFLgFqyII03KsygcAERF2/EArqSIxLsc+vIrsqhvuc8DxwTfw/ljCQEQaABOBv2LVP82B/5YxjhWJYhCR/YCHgEuAlsF1v4m57q665C7Dqq2KrtcEq8r6oQxxlVTa+/xzcP2GMcfvHfP4e2D/3XhNl0SeIFyyNQF+An4Wkc7A7yvhNV8DDhWR34hIbeAqoHVIMT4PXC0ibUWkJXBDaQer6o9Y3foTwLeqOj/YVQ+oC+QCBSJyKnBcOWK4UUSai40TuTxmX2MsCeRiufJCrARR5EegXVFjcRzPAheISDcRqYclsA9VNWGJrBSlvc8rgtu5QcP/cGISE/AYcL2I9BTTSURik6KrBJ4gXLJdB5yPNRo/gn2DDlXwIXw2cC+Qh33znIWN20h2jA9hbQVfAjOwUsCuPIM1Oj8TE/Na4BrgZayhdyCW6MriVqwksxh4Hfh3zHW/AO4HPguOOQiYHnPum8B84EcRia0qKjp/KlYV9HJwfnusXWJ3JHyfg2rAi4AbsfaijrFxquqzwF3BOeuAl7CSjKtEYr8n56oPEUnDqkoGquqHUcfjXFXlJQhXLYhIPxFpFlSL3Axsw75FO+d2kycIV138AliEVVf0A05X1URVTM65MvAqJuecc3F5CcI551xc1WayvlatWmlGRkbUYTjnXJUyc+bMVaoat1t4tUkQGRkZZGdnRx2Gc85VKSKScDYAr2JyzjkXlycI55xzcXmCcM45F1e1aYOIJz8/n5ycHDZv3hx1KFVa/fr1adeuHXXqJJq+xzlXHVXrBJGTk0OTJk3IyMjAJvh05aWq5OXlkZOTQ4cOHXZ9gnOu2qjWVUybN2+mZcuWnhwqQERo2bKll8KcS0FZWZCRAbVq2X1W1q7OKJ9qXYIAPDkkgb+HzqWerCwYPhw2brTnS5bYc4Ahuzv/bgmhliCCCdS+DRY4HxFnf3sReVdEZonIFyJySrA9Q0Q2icjs4PZwmHE651xVM3JkcXIosnGjbU+W0EoQwZTLY7B1eHOAGSIyWVXnxRx2E/C8qj4kIgcDU7CF0AEWqmpp6xM751yNtGmTlRjiWbo0ea8TZgmiN7AgWJR9KzCB4sXbiyjQNHjcDJvDPzJh1OetXbuWBx98sNznnXLKKaxdu7biATjnqo01a2D0aPt8SqT9rhbBLYcwE0RbdlxYPYed1wkehS05mIOVHq6I2dchqHp6X0R+Ge8FRGS4iGSLSHZubm6Fgi2qz1uyBFSL6/MqmiQSJYiCgoJSz5syZQrNmzev2Is756qFwkL44x/tw/+mm+DQQ60qqWHDHY9r2NASSLKEmSDitWyWnFt8MPCkqrYDTgGeFpFaBEsdqmpP4FrgGRFpWuJcVHWsqmaqambr1qUtQWyOPnrnW9Fn95/+FL8+76qr7PGqVTufWxYjRoxg4cKF9OjRg8MOO4xjjjmGc845h65duwJw+umn06tXL7p06cLYsWO3n5eRkcGqVatYvHgxnTt35qKLLqJLly6ceOKJbNq0KeHrPfrooxx22GF0796dM844g43BDzV06FAmTixeHbNx48bbH//tb3+ja9eudO/enREjdmoqcs5FZPlyu69VC+bPh9NOg9mz4fXX4Y47YOxYSE8HEbsfOzZ5DdQQboLIAWIXGW/HzlVIF2ALsKOqnwD1gVaqukVV84LtM4GFwAEhxkpOgiXZ8/Iqdt0777yT/fffn9mzZ/P3v/+dzz77jNGjRzNvnjXFjBs3jpkzZ5Kdnc39999PXpwXnD9/Ppdddhlz586lefPmvPjiiwlf77e//S0zZsxgzpw5dO7cmccff7zU+F5//XUmTZrE9OnTmTNnDtdff33FfmDnXIV98gmcfrqVGBYutG0vvmg1Gt27Fx83ZAgsXmwljMWLk5scINxurjOATiLSAfgBGAScU+KYpcBxwJMi0hlLELki0hpYraoFIrIf0AlbLaxC3nsv8b727eM3+qSn232rVqWfX1a9e/feYcDZ/fffz8svvwzA999/z/z582nZsuUO53To0IEePay9vlevXixevDjh9b/66ituuukm1q5dy4YNGzjppJNKjeett95i2LBhNAzKqnvsscfu/FjOuQpShSlT4K674MMPYY894MYboUUL25+WVvkxhZYgVHWbiFwOvAGkAeNUda6I3AZkq+pk4DrgURG5Bqt+GqqqKiJHAbeJyDagALhYVVeHFStYvV1sn2JIfn0eQKNGjbY/fu+993jrrbf45JNPaNiwIUcffXTcAWn16tXb/jgtLa3UKqahQ4cyadIkunfvzpNPPsl7QVarXbs2hYWFgI2O3rp16/bHPs7BueitWAEDBsDee8M//wkXXAAxNcGRCHUchKpOUdUDVHV/VR0dbLslSA6o6jxVPVJVu6tqD1X9b7D9RVXtEmw/VFVfDTNOsKJZGPV5TZo0Yf369XH3/fTTT7Ro0YKGDRvyzTff8Omnn1bsxYD169fTpk0b8vPzyYppYc/IyGDmzJkAvPLKK+Tn5wNw4oknMm7cuO1tFatXh5qHnXOBn3+G++6D//s/e96mDbz/vlUpXXVV9MkBasBI6vIYMiT5dXgtW7bkyCOP5JBDDqFBgwbstdde2/f169ePhx9+mG7dunHggQfSt2/fCr/e7bffTp8+fUhPT6dr167bk9NFF11E//796d27N8cdd9z2kky/fv2YPXs2mZmZ1K1bl1NOOYW//OUvFY7DORffqlXwwAPwr3/B6tXwy1/Chg2WEA4/POrodiSqJTsWVU2ZmZlackW5r7/+ms6dO0cUUfXi76VzFffOO3DqqTbQ7bTT4IYb4Igjoo1JRGaqama8fV6CcM65EH3xBaxdC0cdBYcdZlVKV14JBx8cdWS75gmiirrsssuYNm3aDtuuuuoqhg0bFlFEzrkiqvDBB9Yj6fXXoXdvmD4dmjSBh6vQzHKeIKqoMWPGRB2Ccy6Od9+1gbfTp0Pr1nD77XDppVFHtXuq9XoQzjkXhpLztj31lLUrACxbBitXwpgxNrbqpptsTENV5CUI55wrh3jrMAwbZoPcnnsOBg2Cs8+G2tXg07Ua/AjOOVd54q3DoGpjGCCaEc9h8Som55wrh0TrLaxcWblxVAZPELHCXuC1DBqnwvBJ59wONm6Ee+6xST0TrbeQzHUYUoVXMRWpjAVenXNVypYtNuXOX/5icyXVqVN587algpqTIK6+2iZST+TTT+2vIdbGjTZj1qOPxj+nRw+bVasUN9xwA+np6Vwa9HMbNWoUIsIHH3zAmjVryM/P54477qB//5KL7e1sw4YN9O/ff6fzFi9ezKmnnspXX30FwN13382GDRsYNWoUCxYs4OKLLyY3N5e0tDReeOEF9t9//12+lnM13bhxMGoUfP+9DXJ7/nmbFqPIyJFW3dS+vSWH6vg9suYkiF0pmRx2tb2MBg0axNVXX709QTz//PNMnTqVa665hqZNm7Jq1Sr69u3LaaedtstZVevXr8/LL7+803mlGTJkCCNGjGDAgAFs3rx5+4yuzrmdFRZaDTPYQLd99rFEcdxxNolnkTDmbUtFNSdB7OKbPhkZiReEqMBCED179mTlypUsW7aM3NxcWrRoQZs2bbjmmmv44IMPqFWrFj/88AM//vgje++9d6nXUlVuvPHGnc5LZP369fzwww8MGDAAsATjnNtZYSG88IKVGLKybEnPBx+EBg12TAw1Tc1JELsSYsXiwIEDmThxIitWrGDQoEFkZWWRm5vLzJkzqVOnDhkZGXHXgSgp0Xmxaz0A269VXSZidC4sqvDKK3DLLfDll9ClS/FHQMn1nmsi78VUJKwFIbBqpgkTJjBx4kQGDhzITz/9xJ577kmdOnV49913WRKv5BJHovP22msvVq5cSV5eHlu2bOG1114DoGnTprRr145JkyYBsGXLlu3rPjhX06la1dGAAbB5MzzzDMyZA7/4RdSRpQ5PELFCWuC1S5curF+/nrZt29KmTRuGDBlCdnY2mZmZZGVlcdBBB5UxvPjn1alTh1tuuYU+ffpw6qmn7nC9p59+mvvvv59u3bpxxBFHsGLFiqT8TM5VVZ98YslBBH77W3jiCZg3DwYPrl6D3JLB14NwZeLvpavqPvoIbr7ZmhQnT4bf/CbqiFJDaetBeAnCOVetzZgB/fpZF9Wvv7ZlPk84IeqoqoZQG6lFpB9wH5AGPKaqd5bY3x54CmgeHDNCVacE+/4EXAAUAFeq6hthxppKvvzyS84777wdttWrV4/p06dHFJFzVdO2bTBwoK3//Pe/27Tb3vhcdqElCBFJA8YAJwA5wAwRmayq82IOuwl4XlUfEpGDgSlARvB4ENAF2Ad4S0QOUNWC8sahqrscX5BqunbtyuzSBvVVsupSDelqhqJSwj//CfXrw6RJ0LGjLdbjyifMKqbewAJVXaSqW4EJQMnhwgo0DR43A5YFj/sDE1R1i6r+D1gQXK9c6tevT15enn/AVYCqkpeX52MoXMpbsADOOw8OOcTGMhR9x+rZ05PD7gqziqkt8H3M8xygT4ljRgH/FZErgEbA8THnflri3LblDaBdu3bk5OSQm5tb3lNdjPr169OuXbuow3COrKydp7gYMMDWeH7ySahbF/7wB/jjH6FVq6ijrfrCTBDx6nVKfpUfDDypqveIyOHA0yJySBnPRUSGA8MB2seZSrFOnTp06NChvHE751JQafNpfvcdXH45jBgBu5iQwJVDmAkiB9g35nk7iquQilwA9ANQ1U9EpD7QqoznoqpjgbFg3VyTFrlzLuXEW6hn40bbvmhR8RxKLnnCfEtnAJ1EpIOI1MUanSeXOGYpcByAiHQG6gO5wXGDRKSeiHQAOgGfhRircy7FJVqoZ+lSTw5hCa0EoarbRORy4A2sC+s4VZ0rIrcB2ao6GbgOeFRErsGqkIaqtSjPFZHngXnANuCy3enB5JyrHrZsgXr1bEqMkqrjQj2polqPpHbOVW1FU2KADW57/33Izy/e37Bh0qZMq7F8JLVzrsr5+GObdvuLL+z5m2/avEkhzKfpEvAE4ZxLKWvXwiWXwJFHQl4erFlTvC+k+TRdAp4gnHMpY+JEOOggKxlcc43NsvqrX0UdVc3lCwY551LGnDnQrh1MmWLVSy5aXoJwzkUmP98m0Zs61Z7ffDNMn+7JIVV4gnDORWL6dDjsMLj+enj1VdtWt64v2pNKPEE45yrVunU2Lcbhh0NuLrz0EjzwQNRRuXg8QTjnKtXLL8ODD8Jll9nU3AMGFI91cKnFG6mdc6H7/nuYO9dWdjvvPGtj6No16qjcrngJwjkXmoICW7inc2f43e9syoxatTw5VBWeIJxzofj8c+jTx8YzHHWUjYyuVy/qqFx5eBWTcy7pFiyA3r2hdWt47jk480xvZ6iKvAThnEuab76x+44d4bHHrBH6rLM8OVRVniCccxX2ww9wxhm2HvRXX9m2oUOhefNIw3IV5AnCObfbCgpgzBhrhJ4yBW6/HQ44IOqoXLJ4G4RzbrcUFMAxx8CHH9paDQ89BPvvH3VULpm8BOGcK5OsLMjIsG6qGRkwYQL07w/jx8Mbb3hyqI48QTjndikrCy66CJYssVXeliyB4cNh771tTQZvhK6ePEE457Zbuxays20MA1gyOOoo+L//g02bdjx240YYObLyY3SVxxOEczWIKqxaBd99V7zt6quhb19o1QpatLAZVm+6yfaJwD772Apu8SxdGn7MLjqhJggR6Sci34rIAhEZEWf/P0RkdnD7TkTWxuwriNk3Ocw4nUtlJev+s7JKP14VVq8ufj52LAwaBJmZlgBat7YuqUVWrIDGjWHgQFub4eWXbXqMIhMm2PrP8bRvv7s/lasKQuvFJCJpwBjgBCAHmCEik1V1XtExqnpNzPFXAD1jLrFJVXuEFZ9zVUFWltX1b9xoz4vq/qF4PeZ33oE337TRy0W3tDRby1kEpk2DGTNs8FqfPtCpk3VLLTJhwq7jGD16xzgAGja07a76CrOba29ggaouAhCRCUB/YF6C4wcDt4YYj3NVzsiRO34ogz0/7zw47TRo0gT++1+45x7Ybz9LAkcdZfeFhZYonnyy4o3IRclo5EirVmrf3pJD0XZXPYmqhnNhkYFAP1W9MHh+HtBHVS+Pc2w68CnQTlULgm3bgNnANuBOVZ0U57zhwHCA9u3b91qyZEkoP4tzUalVy6qM4lm1Clq2hJ9/tknwavuoJrcbRGSmqmbG2xdmG0S87yyJstEgYGJRcgi0D4I+B/iniOzUy1pVx6pqpqpmtm7duuIROxexbdtshbUTTrDxBYnq+NPTLTkANGrkycGFI8wEkQPsG/O8HbAswbGDgGdjN6jqsuB+EfAeO7ZPOFetLFsGf/6zNUKfcQZ8+62VHEaPtrr+WF737ypLmN87ZgCdRKQD8AOWBM4peZCIHAi0AD6J2dYC2KiqW0SkFXAk8LcQY3UuUr/5jY09OOkkW47zlFN2LBV43b+LQmgJQlW3icjlwBtAGjBOVeeKyG1AtqoWdV0dDEzQHRtDOgOPiEghVsq5M7b3k3NV2dq18O9/WxXSW29B06bwwAPW/bRjx52PHzLEE4KLRmiN1JUtMzNTs7Ozow7DuYRmzbIJ7bKyrCdSnz7Ww+igg6KOzNVkpTVSe9OWc5Xgu+/g0EOhQQMrDVxyiT13LpV5gnAuBAsXwiOPwJYtcN99tkbCM8/AySf7Ijqu6vC5mJxLkoICePVVSwKdOsG990JeXvE4hsGDPTm4qsUThHNJMnq0jW7+4gu49VabFmP8eJ8K21VdXsXk3G5QhY8+skbn3/0Ojj/epsQ+5BDrslqnTtQROldxXoJwLoF4s6iuX29JoVs3m/NoypTiKa8zMuC3v/Xk4KoP7+bqXBwlZ1EFG8HcrBksXw49e8Jll9k02o0aRRencxXl3VydK6cbb4w/i2qjRvDpp9C7t7ctuOrPq5hcjVdQYJPkAbz9Nhx+eOKV0latsgFunhxcTeAJwtUoqjZG4bnn4A9/gF/9yrqeTp1q++vVg7p1bZ2FeHwFNVeTeBWTq7ZU4YcfbDW1du1sreVvvoGDD7b99epBjx4wdCi0bWvbfvELeP/9xG0QPouqq0k8QbhqpbAQbr8dsrMtMfz4o23//e8tQRxwgK3RnJkJXbpYaSEeX0HNOe/F5FJQVtauP5jXrrUkUJQI9tzTup+CzYhar54lhMxMu+/WzeZBcs7tyHsxuSqjZNXOkiVw0UV2f+ONtu3MM2HixOJzOna09ROKzJuXuGTgnCs7L0G4lJKRYcmgJBGb+K5OHXjsMcjNtdJBZia0aFHpYTpXbXgJwqW0rVvh44+tKihR99JYF14YfkzOOU8QLiKLF1vX0qlTbezBhg02E2r79vFLEO3b+xQWzlU2TxCuUmzebPMYtW4Nc+fapHYA6elw7rnQrx8cfbQ1SHv3UudSgycIF5r5862E8Prr8N57lgjGjrVxCA88AMcdBwceuOOoZO9e6lzq8EZqlzQFBZCWZo+POcaSAtjiOf36wRln2Mhl51zqiKyRWkT6AfcBacBjqnpnif3/AI4JnjYE9lTV5sG+84Gbgn13qOpTYcbqyk8Vvv66uJSwYAEsWmQlgoED7davH+y/f9SROud2R2gJQkTSgDHACUAOMENEJqvqvKJjVPWamOOvAHoGj/cAbgUyAQVmBueuCSteVz7PPgsjRhT3Ojr4YCshbNpkbQaXXRZtfM65igtzsr7ewAJVXaSqW4EJQP9Sjh8MPBs8Pgl4U1VXB0nhTaBfiLE64i+Qowpz5sCdd1oj8syZdmzLltCrl7UpLFliDc93323JwTlXPYRZxdQW+D7meQ7QJ96BIpIOdADeKeXctnHOGw4MB2jv02xWSLwRzOefD5deCuvW2bYePWyKC4ATT7Sbc676CjNBxJsxP1GL+CBgoqoWlOdcVR0LjAVrpN6dIJ0ZOXLnBXKK1kl44gk46SRo0yaa2Jxz0QiziikH2DfmeTtgWYJjB1FcvVTec10FFRbGH5wG1qYwdKgnB+dqojATxAygk4h0EJG6WBKYXPIgETkQaAF8ErP5DeBEEWkhIi2AE4NtLslmz4Yjj0y832vunKu5QksQqroNuBz7YP8aeF5V54rIbSJyWsyhg4EJGjMgQ1VXA7djSWYGcFuwzSXRrFnW0Lxwoa2XULKB2UcwO1ezlWmgnIj0Beaq6vrgeRPgYFWdHnJ8ZeYD5cpG1UY4H3CAPb7vPmuMbtGibOswOOeql9IGypU1QcwCDi36li8itYBsVT00qZFWgCeIXfv2Wxuf8Omn9rjtTv3CnHM1TWkJoqxVTFKiCqgQn8epyti4EW66Cbp2tRXY/vY32HvvqKNyzlVYvMFLSVTWD/lFInIlECzqyKXAoqRG4kKxYQN0725TYJx3Hvz977DXXlFH5ZyrsHiDl4YPt8dJqhsuawniYuAI4AeKB7wNT0oELhQ//WT3jRvDsGHw7rvw7397cnCu2og3eGnjRtueJD6bazWzdSvce681ML/7ri3J6ZyrhmrVsp4mJYnY4KYyqvBsriLyBPFHMv+uzFG40L37rk2N8c03MGCAlxacq7ZWrYL69W0ka0lJHLxU1iqm14D/BLe3gabAhqRF4Sps+HA49ljYsgVeew1eegn23XfX5znnqpj337eJ0bZu3Xkd3iQPXipTglDVF2NuWcBZwCFJi8LtloKC4hLmAQfAzTfbrKq//nW0cTnnQlBQAH/+s30TbNAAPvvMJkpLT7dqpfR0m145iYOXdqsNIpge4z+q2jFpkVRQTWuD+OwzuOQSa4/67W+jjsY5F6qcHFuz9/337f7BB6FJk6RcusLjIERkvYisC24/Aa8C1yclOlcuq1fDxRdD376wfPnOJUznXDXz2mtWpZSdDU89BU8/nbTksCtlaqRW1SbBKm+dgPpFm0OLysX1wgvWCL1mDVx9NYwaBU2bRh2Vcy4UW7bYso3//KcNZnruOTjwwEoNoay9mC4ErsKm3Z4N9MVmXz02vNBcSfn51tbw4IP29+Kcq6bmz4dBg+Dzz+GKK2z6g/r1d31ekpW1F9NVwGHAElU9Bls7Oje0qBwA69fDH/5gE+oBDB4MH37oyaHShDyNgXNxZWXBoYfC//4HkybB/fdHkhyg7Alis6puBhCReqr6DVC5ZZ1qLN7n0MSJ0Lkz3HMPLF5sx4nYMa4SFE1jsGSJdRUrmsbAk4QLy4YNNu3Buedam8OcOdC/f6QhlXU215eBYcDVWLXSGqCOqp4SbnhlV1V7MZWcTgUsCRQW2t/IQw9Zg7SrZBkZ8ZfZS08vztjOJcvs2Val9N13NrPmLbdA7cqZD7XCvZhUdYCqrlXVUcDNwOPA6ckLseYaORL6b8zif2RQQC3+RwZnF2bRogXMmOHJITJLl5Zvu3O7QxXGjLF/9HXr4O234bbbKi057Eq5o1DV98MIpKY6ckkWYxlOI6wIkcESHmU4w9dA7dq+Wk9kWreGlSt33t6kic2E2KxZ5cfkqpfVq+GCC6yd4ZRT4Mkn7e8uhXiNdsTuShu5PTkUacRG7kq7MaKIHO+/bzhn4gsAABl9SURBVH2JRXbcnpZm3/I6drRvffn50cTnqr5p06wO+T//sYbGV19NueQAniAi17YgfpVF24Kl1v957dpKjqiGmzbN5iopSgKx0xg89ZQNVjrkELj8crufNCn+jJrOxVNQYHMl/epXNsp12jS49trU7X2iqqHdgH7At8ACYESCY84C5gFzgWdithdgYy5mA5N39Vq9evXSKqWwUPWhh1RFVO0jZsdb3bp237Ch6vDhqnPmRB1x9ffpp6pNmqgecIDqsmWJjyssVH31VdWDDrLf0S9+oTp9euXF6aqmH35QPfZY+5sZPFj1p5+ijkhVVbHlo+N/hifaUdEbkAYsBPYD6gJzgINLHNMJmAW0CJ7vGbNvQ3ler0oliJUrVX/zG1XQgi6H6EYa7JgcGjZUHT9e9fPPVS+4QLVBg+IPomefVd2yJeqfoPqZMUO1WTPV/fdXzckp2zn5+aoPP6y65572+xk0SHXRonDjdFXTlCmqrVvb//a4cfYlI0VElSAOB96Ief4n4E8ljvkbcGGC86tngpgyRXWvvayE8I9/6KuvFOhgxuvPrdOtNJGebskhVl6e6t1324cXqO69t+ott5T9g8yVbtYs1RYtVDMyVJcsKf/569ap3nyzJfK6dVWvu0519erkx+mqni1bVP/wB/u/7dpVdd68qCPaSVQJYiDwWMzz84AHShwzKUgS04BPgX4x+7YB2cH20xO8xvDgmOz27duH9w4mw8aNqpdfbm/5IYdsrzJasUL1nntUt24twzUKCizB/PrXlkzS0lQHDlR9992U+kZSpXzxhWrLlqr77lvxb/85OarDhtnvpkUL1XvvVd28OTlxOjN+vH2JSvRlKpUsXKh62GH2P3/ppfYZkIKiShBnxkkQ/ypxzGvAy0AdoAO23nXzYN8+wf1+wGJg/9JeL6VLELNmqR58sL3dV12lumlTxa+5cKHqH/+ouscedt0uXVTHjLFvs65s5s61Yv8++6jOn5+8686erXriifZ72W8/1eef9wSeDOPHWxVNvOrYVDNhgrVnNW+u+uKLUUdTqlSuYnoYGBrz/G3gsDjXehIYWNrrpWSCKCiwqqG6da1a6I03dtj9wguqzzxTwc+OjRutTrNXL/t1NmliJZUULMqmlG++sd/J3nvb4zBMnWrVCqDat6/qRx+F8zo1RXr6jsmh6LbnnqoLFqRG29zPP6teeKHFdfjhqosXRx3RLkWVIGoDi4KSQVEjdZcSx/QDngoetwK+B1oCLYB6Mdvnl2zgLnlLuQSRk6N63HH2Fp9+umpu7g67CwutE8wRRyTp9QoLrRfOeecV94A69lj79pKfn6QXqSbmz7dSQ+vWVooI07Ztqo8/rtqmjf1OzjgjuaWVmiRRj7+im4j9Xg8/3DoM3HCD6oMPqv7nP6pffhl+6fqLL1Q7d7Y4bryxjPXG0YskQdjrcgrwXdCbaWSw7TbgtOCxAPdi3Vy/BAYF248Ins8J7i/Y1WulVIJ44QWrg27YUPXRR+MWEaZNs3f/8cdDeP2VK1X/+lfV9u3tRdq1U739dmvwqOkWLbL2hpYt7R+6smzYoHrbbaqNGqnWrq165ZU7fWlwCaxdq3r99YkTw557Win61ltVhw5VPeYY69BRp87Ox7Zoodqjh2r//vY7uOce1YkTrRfbypVlL87HtoW0b29tT/XrW4n0zTfDfDeSLrIEUZm3lEgQ69bZHyhY49R33yU89He/U23cWHX9+hDj2bZNddIk1RNOsJjq1FE95xzLTjWxTnzJEuup1KKFtQtFYflyG9dSq5Z1q73rruS0SVVH+fk2Vqh1a/v7/eUvi7t8l6UNoqDAxh58/LF1D7/zTtVLLlE95RRrs2vceOcE0qCB6oEHWhvSRRep3nGH6tNPq37wgf395OfHbwsp6qVUBb+EeYKoDJ98Yg2StWqpjhxZavFy3Tr7InnBBZUY3zffWAN5s2b2a+/Rw0o3P/9ciUFEKCfHvlU2a6aanR11NKpffWW90cC+gWZl2QeaM2+8Yb39ihJD0e8smb2YCgutO/KsWfZF6r77VK+91qoBMzOLE1PsLS3NbvFKMqnekzIBTxBhys9XHTXK/mjS0+2bxi589ZV9gfn44/DD28n69Ta4q6jxtHlz1WuusXrxqtSFsDyWLbPR0U2aWDtNKnn7bdWePe130auXdVmuyb7+2r7hg2qHDlb9E2Vp9+ef7cvVG2+ojh1rX/5KawOpgjxBhGXhQmsQA9Vzz7W60jIqLIy4lqew0JLZ2WdbnThY6acqdCEsjxUrrOGwUaPU7UVUUKD6739bWxHYKPua1gtt1SrrfZeWptq0qerf/pa6Y0gS9aZKT486st3iCSLZCgtVn3jC6jCbNbO+qmWUl2ftlSll2bLiqqdq8kevqtYIfMghVq/83ntRR7NrGzda54ImTeyD8uKLLcFV15KdqnVNvfdeK8nWqmU/848/Rh1V6arSeIwy8ASRTHl5NnoZVI86qtz9nK+7zsa2pVzVf6IuhCJVs0E7L8/aWerXV33rraijKZ+VK+3bdO3aqvXq7dwbpwp/GG1XWGj1/p062c900klW91pVVKOk7QkiWd5+W7VtW/vH/etfrZdQOWzZYu1eAwaEFF9FJCo2FzUSRtJgspvWrLH6/Lp1bbBaVfXttzv32qkOJbtZs6wrKlj135QpUUdUo5WWIFJ0EvIUs2UL/PGPcPzx0LgxfPopjBhhC8iUw2uvQW6uLSKVckaPhoYNd9zWsCGcf76tk3vEEXD66TBvXjTxldW6ddCvH3zxBbz0Epx0UtQR7b4DDoDNm+PvW7IEpk5NvD8VLV8OF14Ihx5qv58HHoA5c+Dkk6OOzCWSKHNUtVtoJYi5c62qAqx+tAINCKecYgM9U3Zgc6Ji8/r11h+8aVOrJx46dPdmPQ3b+vU2NL12bau+qA4SleyKqgQbNbIi6bhxqVt3v3Gj/f00amTVZdde67PdphC8imk3FBaq/utfVofdqpXqK69U6HLLl9tn6403Jim+KOTm2j933bpWN37ttdb7JBVs2GBtQmlpNpK9ukjUIPrEE1Y1c/HFVu1ZlDT69lUdPdqmloi67aiw0DpwFI3oHzDApxlJQZ4gymvFiuK+2P362ad7Enz+eTVZwmHJEitF1KplpYo77oi2a9bGjTbvVK1a5epRVmXsqkG0sND+uP78ZxvgVZRIMjJUr7hC9b//rfyJ7D75xJIV2DiPmj6+I4V5giiPV1+1luR69awEEfW3sFT21Vc2pw3YIkhjxlT+BGWbNtm0CCI2lsDZ9BKPPKJ66qlWAgbrOjtwoL1HYZb6liyx5TTB5iUaN67cnTlc5fIEUZrYb2dFc7N065bULncffqh6/vlVcpqWspk2zXo6gU1n8eyzlTNtxObNxSW9UGY9rAZ+/ll18mSbV6hoRtlatWz52rvusgF5yfgStH69jTKuX99uN98c8kRjLlk8QSQSr363dm3VJ58s/7VKce65Ng4tRReUSo7CQtXXXiuewqNnT5ueIKwS2NatxaWXhx8O5zWqm4ICm7X05puLO14UJfWrr1Z9553ylwCLpjPfe2+71pAhqkuXhhO/C4UniEQqYcj8mjX2heqSS5J2ydS2bZvNfpmRYe/lMceoTp+e3NfIz7cJ1cCqAd3uWbrU1ks4+eTiNUSaNbO1FLKydu5pVLIt5MYbixPN4Yen3jxXrkw8QSRS2ujhJHnwQbtkKkwgWqk2b7bZMYtmxDzjjOSs3Jafbx9gYFM0uORYv171pZdsXYM997T3Ny1N9eijbc2Eu++OP8V1y5a2vKa31VVZpSUIsf1VX2ZmpmZnZ5fvpIwMG3BUUno6LF6cjLDIzIRt22DWLBBJyiWrlvXr4Z577LZpE/zud3DrrdC2bfmvVVAAw4bB00/DXXfB9dcnP14HhYXw2Wfw6qt2+/LLxMfuuy8sXVp5sbmkE5GZqpoZb1/NHkmdaPTw6NFJuXx+Phx1FFx9dQ1NDgBNmsCoUbBwIVx2GTz5JHTsCDfcAGvWlP06hYVw0UWWHG6/3ZNDmGrVgr597f/giy/gf/9LfGxOTuXF5SpfoqJFVbslpRdTFZ90q0pYtMha7UVsBs8779z1zIWFhbYKG6jeckvlxOl2VM2muHbF8LmYSjFkiFUnFRba/ZAhSbnsli3w9tt2WRejQwcrBcyebfM7jRgBnTrBo49aXVxJqnDFFTB2rB07alSlh+wIvbTtUpMniJBMmmRz+733XtSRpKhu3eA//4H337c2n+HDoUsXmDgRsrKsfahWLWjWDMaMgeuug7/8pQbX1UVsyBBL0unp9jtIT7fnSfpC5VJTqAlCRPqJyLciskBERiQ45iwRmScic0XkmZjt54vI/OB2fphxhuHxx6F9ezj66KgjSXFHHQXTpllGrV0bzjwT/u//rPOAqjVy164NPXt6cohaSKVtl7pCSxAikgaMAU4GDgYGi8jBJY7pBPwJOFJVuwBXB9v3AG4F+gC9gVtFpEVYsSbb4sXw1lvW4aaWl9F2TQT697cG0ZYtd66X27YNRo6MJjbnarAwP756AwtUdZGqbgUmAP1LHHMRMEZV1wCo6spg+0nAm6q6Otj3JtAvxFiT6okn7H7YsGjjqHLS0mD16vj7vCulc5UuzATRFvg+5nlOsC3WAcABIjJNRD4VkX7lOBcRGS4i2SKSnZubm8TQK2bqVGt/SE+POpIqqH378m13zoUmzAQRr8K45Ki82kAn4GhgMPCYiDQv47mo6lhVzVTVzNatW1cw3OT56CN46qmoo6iivLeMcykjzASRA+wb87wdsCzOMa+oar6q/g/4FksYZTk3JalCnTrQpk3UkVRR3lvGuZQRZoKYAXQSkQ4iUhcYBEwuccwk4BgAEWmFVTktAt4AThSRFkHj9InBtpSWl2dd+qdMiTqSKs57yziXEmqHdWFV3SYil2Mf7GnAOFWdKyK3YSP3JlOcCOYBBcAfVTUPQERux5IMwG2qmqD1MnWMH28zSrRrF3UkzjlXcTV7sr4kUoXu3aFePZgxY9fHO+dcKvDJ+ipBdrZNennBBVFH4pxzyeEJIknGjYMGDWDw4Kgjcc655AitDaKmGTgQDjnEpg5yzrnqwBNEkhx3nN2cc6668CqmJHjoIZg/P+oonHMuuTxBVNCCBXDppfDCC1FH4pxzyeUJooLGjbMZW8+vchOSO+dc6TxBVMC2bbbE8sknQ9udphJ0zrmqzRNEBUydCsuX+9gH51z15AmiAhYssFmoTz016kiccy75PEFUwNVXW5KoUyfqSJxzLvk8Qeymn36ye08OzrnqyhPEblCFPn3g4oujjsQ558LjCWI3fPwxfPst9O4ddSTOORceTxC7Ydw4aNwYzjor6kiccy48niDKaf16eO45OPtsSxLOOVddeYIop+efh59/9rEPzrnqz2dzLaczz4T69aFv36gjcc65cHmCKKemTWHIkKijcM658IVaxSQi/UTkWxFZICIj4uwfKiK5IjI7uF0Ys68gZvvkMOMsqwcegEceiToK55yrHKGVIEQkDRgDnADkADNEZLKqzitx6HOqenmcS2xS1R5hxVdeW7fCbbfBL34Bv/991NE451z4wixB9AYWqOoiVd0KTAD6h/h6oXrtNcjN9cZp51zNEWaCaAt8H/M8J9hW0hki8oWITBSRfWO21xeRbBH5VERODzHOMnn8cdhnHzjppKgjcc65yhFmgpA427TE81eBDFXtBrwFPBWzr72qZgLnAP8Ukf13egGR4UESyc7NzU1W3Dv54Qeb2nvoUKjtzfrOuRoizASRA8SWCNoBy2IPUNU8Vd0SPH0U6BWzb1lwvwh4D+hZ8gVUdayqZqpqZuvWrZMbfYxVq+CII2DYsNBewjnnUk6YCWIG0ElEOohIXWAQsENvJBFpE/P0NODrYHsLEakXPG4FHAmUbNyuNN27w4cfQseOUUXgnHOVL7QKE1XdJiKXA28AacA4VZ0rIrcB2ao6GbhSRE4DtgGrgaHB6Z2BR0SkEEtid8bp/VQpliyBhg0hxAKKc86lJFEt2SxQNWVmZmp2dnbSr3veefDGG7Bsmbc/OOeqHxGZGbT37sTnYirF2rUwcSIMHOjJwTlX83iCKMWzz8LmzT72wTlXM3mCKMXjj0O3bnDooVFH4pxzlc8TRAILF8LMmVZ6kHgjOpxzrprzmvUE9t/fkkTLllFH4pxz0fAEUYr99os6Aueci45XMcXx4ovQvz/k5UUdiXPORcdLEHE88gh89x20aBF1JM45Fx0vQZSwZAm89ZbNu1TL3x3nXA3mH4ElPPGE3Q8dGmkYzjkXOU8QMQoKYNw4OP54SE+POhrnnIuWt0HE2LoVhg+HXr12faxzzlV3niBiNGgAN90UdRTOOZcavIopkJdnE/Nt3Rp1JM45lxo8QQTGj4czz4Rvv406EuecSw2eIABVm5jvsMOga9eoo3HOudRQ4xNEVhbssw98+aXNvZSVFXVEzjmXGmp0I3VWlvVa2rjRnq9ebc8BhgyJLi7nnEsFNboEMXJkcXIosnGjbXfOuZquRieIpUvLt90552qSUBOEiPQTkW9FZIGIjIizf6iI5IrI7OB2Ycy+80VkfnA7P4z42rcv33bnnKtJQksQIpIGjAFOBg4GBovIwXEOfU5VewS3x4Jz9wBuBfoAvYFbRSTpc6uOHg0NG+64rWFD2+6cczVdmCWI3sACVV2kqluBCUD/Mp57EvCmqq5W1TXAm0C/ZAc4ZAiMHWvzLonY/dix3kDtnHMQbi+mtsD3Mc9zsBJBSWeIyFHAd8A1qvp9gnPbljxRRIYDwwHa72a90JAhnhCccy6eMEsQEmeblnj+KpChqt2At4CnynEuqjpWVTNVNbN169YVCtY559yOwkwQOcC+Mc/bActiD1DVPFXdEjx9FOhV1nOdc86FK8wEMQPoJCIdRKQuMAiYHHuAiLSJeXoa8HXw+A3gRBFpETROnxhsc845V0lCa4NQ1W0icjn2wZ4GjFPVuSJyG5CtqpOBK0XkNGAbsBoYGpy7WkRux5IMwG2qujqsWJ1zzu1MVHeq2q+SMjMzNTs7O+ownHOuShGRmaqaGXdfdUkQIpILLIk6jgpqBayKOogU4u/Hjvz9KObvxY4q8n6kq2rcXj7VJkFUByKSnSiT10T+fuzI349i/l7sKKz3o0bPxeSccy4xTxDOOefi8gSRWsZGHUCK8fdjR/5+FPP3YkehvB/eBuGccy4uL0E455yLyxOEc865uDxBpAAR2VdE3hWRr0VkrohcFXVMURORNBGZJSKvRR1L1ESkuYhMFJFvgr+Rw6OOKUoick3wf/KViDwrIvWjjqkyicg4EVkpIl/FbNtDRN4MFlh7M1nr53iCSA3bgOtUtTPQF7gsweJKNclVFM/NVdPdB0xV1YOA7tTg90VE2gJXApmqegg2jc+gaKOqdE+y8/o4I4C3VbUT8HbwvMI8QaQAVV2uqp8Hj9djHwA7rX9RU4hIO+DXwGNRxxI1EWkKHAU8DqCqW1V1bbRRRa420EBEagMNqWEzPavqB9jcdbH6U7xcwlPA6cl4LU8QKUZEMoCewPRoI4nUP4HrgcKoA0kB+wG5wBNBldtjItIo6qCioqo/AHcDS4HlwE+q+t9oo0oJe6nqcrAvnMCeybioJ4gUIiKNgReBq1V1XdTxREFETgVWqurMqGNJEbWBQ4GHVLUn8DNJqj6oioK69f5AB2AfoJGInBttVNWXJ4gUISJ1sOSQpaovRR1PhI4EThORxdg65seKyPhoQ4pUDpCjqkUlyolYwqipjgf+p6q5qpoPvAQcEXFMqeDHovV1gvuVybioJ4gUICKC1TF/rar3Rh1PlFT1T6raTlUzsMbHd1S1xn5DVNUVwPcicmCw6ThgXoQhRW0p0FdEGgb/N8dRgxvtY0wGzg8enw+8koyLhrZgkCuXI4HzgC9FZHaw7UZVnRJhTC51XAFkBSszLgKGRRxPZFR1uohMBD7Hev/NooZNuyEizwJHA61EJAe4FbgTeF5ELsCS6JlJeS2fasM551w8XsXknHMuLk8Qzjnn4vIE4ZxzLi5PEM455+LyBOGccy4uTxDO7YKIFIjI7Jhb0kYyi0hG7KyczqUSHwfh3K5tUtUeUQfhXGXzEoRzu0lEFovIXSLyWXDrGGxPF5G3ReSL4L59sH0vEXlZROYEt6IpItJE5NFgjYP/ikiD4PgrRWRecJ0JEf2YrgbzBOHcrjUoUcV0dsy+daraG3gAm4WW4PG/VbUbkAXcH2y/H3hfVbtj8ynNDbZ3AsaoahdgLXBGsH0E0DO4zsVh/XDOJeIjqZ3bBRHZoKqN42xfDByrqouCyRZXqGpLEVkFtFHV/GD7clVtJSK5QDtV3RJzjQzgzWChF0TkBqCOqt4hIlOBDcAkYJKqbgj5R3VuB16CcK5iNMHjRMfEsyXmcQHFbYO/BsYAvYCZwQI5zlUaTxDOVczZMfefBI8/pngZzCHAR8Hjt4FLYPua200TXVREagH7quq72OJJzYGdSjHOhcm/kTi3aw1iZtkFWx+6qKtrPRGZjn3ZGhxsuxIYJyJ/xFaDK5p99SpgbDDjZgGWLJYneM00YLyINAME+IcvNeoqm7dBOLebgjaITFVdFXUszoXBq5icc87F5SUI55xzcXkJwjnnXFyeIJxzzsXlCcI551xcniCcc87F5QnCOedcXP8PdOPhgUu00oUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:37.101509Z",
     "start_time": "2020-12-28T14:41:37.088545Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:38.266436Z",
     "start_time": "2020-12-28T14:41:38.248529Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:51.028065Z",
     "start_time": "2020-12-28T14:41:50.987174Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的保存与使用\n",
    "torch.save(model, './model/NFM.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:52.416325Z",
     "start_time": "2020-12-28T14:41:52.394384Z"
    }
   },
   "outputs": [],
   "source": [
    "net_clone = torch.load('./model/NFM.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:53.474494Z",
     "start_time": "2020-12-28T14:41:53.460533Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T14:41:54.286799Z",
     "start_time": "2020-12-28T14:41:54.270770Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
